diff options
| author | Ondřej Surý <ondrej@sury.org> | 2011-06-30 15:34:22 +0200 |
|---|---|---|
| committer | Ondřej Surý <ondrej@sury.org> | 2011-06-30 15:34:22 +0200 |
| commit | d39f5aa373a4422f7a5f3ee764fb0f6b0b719d61 (patch) | |
| tree | 1833f8b72a4b3a8f00d0d143b079a8fcad01c6ae | |
| parent | 8652e6c371b8905498d3d314491d36c58d5f68d5 (diff) | |
| download | golang-d39f5aa373a4422f7a5f3ee764fb0f6b0b719d61.tar.gz | |
Imported Upstream version 58upstream/58
617 files changed, 37339 insertions, 17045 deletions
@@ -12,6 +12,7 @@ Abhinav Gupta <abhinav.g90@gmail.com> Adrian O'Grady <elpollouk@gmail.com> Albert Strasheim <fullung@gmail.com> Alex Brainman <alex.brainman@gmail.com> +Alexander Orlov <alexander.orlov@loxal.net> Alexey Borzenkov <snaury@gmail.com> Amrut Joshi <amrut.joshi@gmail.com> Andrei Vieru <euvieru@gmail.com> @@ -43,18 +44,22 @@ Daniel Theophanes <kardianos@gmail.com> Dave Cheney <dave@cheney.net> David Forsythe <dforsythe@gmail.com> David G. Andersen <dave.andersen@gmail.com> +David Jakob Fritz <david.jakob.fritz@gmail.com> David Titarenco <david.titarenco@gmail.com> Dean Prichard <dean.prichard@gmail.com> Devon H. O'Dell <devon.odell@gmail.com> Dmitry Chestnykh <dchest@gmail.com> Eden Li <eden.li@gmail.com> +Eivind Uggedal <eivind@uggedal.com> Eoghan Sherry <ejsherry@gmail.com> Eric Clark <zerohp@gmail.com> Eric Eisner <eric.d.eisner@gmail.com> Evan Shaw <chickencha@gmail.com> +Fan Hongjian <fan.howard@gmail.com> Fazlul Shahriar <fshahriar@gmail.com> Firmansyah Adiputra <frm.adiputra@gmail.com> Florian Uekermann <florian@uekermann-online.de> +Gary Burd <gary@beagledreams.com> Giles Lean <giles.lean@pobox.com> Google Inc. Graham Miller <graham.miller@gmail.com> @@ -69,9 +74,11 @@ James Toy <nil@opensesame.st> James Whitehead <jnwhiteh@gmail.com> Jan H. Hosang <jan.hosang@gmail.com> Jan Mercl <befelemepeseveze@gmail.com> +Jeff Hodges <jeff@somethingsimilar.com> Jeff R. Allen <jra@nella.org> Jim McGrath <jimmc2@gmail.com> Joe Poirier <jdpoirier@gmail.com> +Jonathan Mark <jhmark@xenops.com> Jonathan Wills <runningwild@gmail.com> Jose Luis Vázquez González <josvazg@gmail.com> Josh Goebel <dreamer3@gmail.com> diff --git a/CONTRIBUTORS b/CONTRIBUTORS index c2ebaf32b..a395c11cc 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -36,6 +36,7 @@ Adam Langley <agl@golang.org> Adrian O'Grady <elpollouk@gmail.com> Albert Strasheim <fullung@gmail.com> Alex Brainman <alex.brainman@gmail.com> +Alexander Orlov <alexander.orlov@loxal.net> Alexey Borzenkov <snaury@gmail.com> Amrut Joshi <amrut.joshi@gmail.com> Andrei Vieru <euvieru@gmail.com> @@ -80,6 +81,7 @@ David Anderson <danderson@google.com> David Crawshaw <david.crawshaw@zentus.com> David Forsythe <dforsythe@gmail.com> David G. Andersen <dave.andersen@gmail.com> +David Jakob Fritz <david.jakob.fritz@gmail.com> David Symonds <dsymonds@golang.org> David Titarenco <david.titarenco@gmail.com> Dean Prichard <dean.prichard@gmail.com> @@ -87,14 +89,17 @@ Devon H. O'Dell <devon.odell@gmail.com> Dmitriy Vyukov <dvyukov@google.com> Dmitry Chestnykh <dchest@gmail.com> Eden Li <eden.li@gmail.com> +Eivind Uggedal <eivind@uggedal.com> Eoghan Sherry <ejsherry@gmail.com> Eric Clark <zerohp@gmail.com> Eric Eisner <eric.d.eisner@gmail.com> Evan Shaw <chickencha@gmail.com> +Fan Hongjian <fan.howard@gmail.com> Fazlul Shahriar <fshahriar@gmail.com> Firmansyah Adiputra <frm.adiputra@gmail.com> Florian Uekermann <florian@uekermann-online.de> <f1@uekermann-online.de> Fumitoshi Ukai <ukai@google.com> +Gary Burd <gary@beagledreams.com> <gary.burd@gmail.com> Giles Lean <giles.lean@pobox.com> Graham Miller <graham.miller@gmail.com> Gustavo Niemeyer <gustavo@niemeyer.net> <n13m3y3r@gmail.com> @@ -113,10 +118,14 @@ James Whitehead <jnwhiteh@gmail.com> Jamie Gennis <jgennis@google.com> Jan H. Hosang <jan.hosang@gmail.com> Jan Mercl <befelemepeseveze@gmail.com> +Jeff Hodges <jeff@somethingsimilar.com> Jeff R. Allen <jra@nella.org> <jeff.allen@gmail.com> Jim McGrath <jimmc2@gmail.com> Joe Poirier <jdpoirier@gmail.com> +Johan Euphrosine <proppy@google.com> John DeNero <denero@google.com> +Jonathan Allie <jonallie@google.com> +Jonathan Mark <jhmark@xenops.com> <jhmark000@gmail.com> Jonathan Wills <runningwild@gmail.com> Jos Visser <josv@google.com> Jose Luis Vázquez González <josvazg@gmail.com> @@ -145,6 +154,7 @@ Maxim Ushakov <ushakov@google.com> Micah Stetson <micah.stetson@gmail.com> Michael Elkins <michael.elkins@gmail.com> Michael Hoisie <hoisie@gmail.com> +Michael T. Jones <mtj@google.com> <michael.jones@gmail.com> Miek Gieben <miek@miek.nl> <remigius.gieben@gmail.com> Mikael Tillenius <mikti42@gmail.com> Mikio Hara <mikioh.mikioh@gmail.com> @@ -202,6 +212,7 @@ Vincent Vanackere <vincent.vanackere@gmail.com> Vinu Rajashekhar <vinutheraj@gmail.com> Vish Subramanian <vish@google.com> Wei Guangjing <vcc.163@gmail.com> +William Chan <willchan@chromium.org> William Josephson <wjosephson@gmail.com> Yasuhiro Matsumoto <mattn.jp@gmail.com> Yongjian Xu <i3dmaster@gmail.com> diff --git a/doc/GoCourseDay1.pdf b/doc/GoCourseDay1.pdf Binary files differindex c2c8c4a7d..5a7749c18 100644 --- a/doc/GoCourseDay1.pdf +++ b/doc/GoCourseDay1.pdf diff --git a/doc/all.css b/doc/all.css index e496388ef..a985d8ffb 100644 --- a/doc/all.css +++ b/doc/all.css @@ -140,7 +140,7 @@ h1#title { } #content h2 { border-top: 2px solid #ddd; - padding: 8px 5px; + padding: 8px 0; margin: 1.5em 0 0; } #content .subtitle { @@ -198,3 +198,8 @@ span.alert { padding: 0.25em 1em; background: #F4F4F4; } +sup.new { + color: red; + font-size: 8px; + line-height: 0; +} diff --git a/doc/codelab/wiki/get.go b/doc/codelab/wiki/get.go index 342831416..c36684e3e 100644 --- a/doc/codelab/wiki/get.go +++ b/doc/codelab/wiki/get.go @@ -37,7 +37,7 @@ func main() { b := strings.NewReader(*post) r, err = http.Post(url, "application/x-www-form-urlencoded", b) } else { - r, _, err = http.Get(url) + r, err = http.Get(url) } if err != nil { log.Fatal(err) diff --git a/doc/codelab/wiki/index.html b/doc/codelab/wiki/index.html index d059fa027..cc187ad90 100644 --- a/doc/codelab/wiki/index.html +++ b/doc/codelab/wiki/index.html @@ -501,7 +501,7 @@ This will prevent user data from corrupting the form HTML. </p> <p> -Now that we've removed the <code>fmt.Sprintf</code> statement, we can remove +Now that we've removed the <code>fmt.Fprintf</code> statement, we can remove <code>"fmt"</code> from the <code>import</code> list. </p> diff --git a/doc/codelab/wiki/wiki.html b/doc/codelab/wiki/wiki.html index 3628eeb56..4db880b9d 100644 --- a/doc/codelab/wiki/wiki.html +++ b/doc/codelab/wiki/wiki.html @@ -428,7 +428,7 @@ This will prevent user data from corrupting the form HTML. </p> <p> -Now that we've removed the <code>fmt.Sprintf</code> statement, we can remove +Now that we've removed the <code>fmt.Fprintf</code> statement, we can remove <code>"fmt"</code> from the <code>import</code> list. </p> diff --git a/doc/contrib.html b/doc/contrib.html index 9d0c42726..84d2cda6a 100644 --- a/doc/contrib.html +++ b/doc/contrib.html @@ -4,7 +4,7 @@ <h2 id="howto">How you can help</h2> -<h3>Reporting issues</h3> +<h3><a href="http://code.google.com/p/go/issues">Reporting issues</a></h3> <p> If you spot bugs, mistakes, or inconsistencies in the Go project's code or @@ -19,7 +19,7 @@ a new one.) We pride ourselves on being meticulous; no issue is too small. </p> -<h3>Contributing code</h3> +<h3><a href="contribute.html">Contributing code</a></h3> <p> Go is an open source project and we welcome contributions from the community. @@ -39,7 +39,7 @@ are particularly in need of outside help. <div class="right-column"> -<h2 id="">The Go Project</h2> +<h2 id="the_go_project">The Go Project</h2> <h3 id="build_status"><a href="http://godashboard.appspot.com/">Build Status</a></h3> <p>View the status of Go builds across the supported operating @@ -49,7 +49,10 @@ systems and architectures.</p> <p>Features and ideas being developed or discussed by the Go team.</p> <h3 id="release"><a href="devel/release.html">Release History</a></h3> -<p>A summary of the changes between tagged releases of Go.</p> +<p>A summary of the changes between Go releases.</p> + +<h3 id="release"><a href="devel/weekly.html">Weekly Snapshot History</a></h3> +<p>A summary of the changes between weekly snapshots of Go.</p> <h3 id="golang-dev"><a href="http://groups.google.com/group/golang-dev">Developer Mailing List</a></h3> <p>The <a href="http://groups.google.com/group/golang-dev">golang-dev</a> diff --git a/doc/contribute.html b/doc/contribute.html index ba70c9600..c4b1ce2b5 100644 --- a/doc/contribute.html +++ b/doc/contribute.html @@ -209,7 +209,8 @@ The <code>Reviewer</code> line lists the reviewers assigned to this change, and the <code>CC</code> line lists people to notify about the change. These can be code review nicknames or arbitrary email addresses. -If you don't know who is best to review the change, set the +Unless explicitly told otherwise, such as in the discussion leading +up to sending in the change list, set the reviewer field to the <a href="http://groups.google.com/group/golang-dev">golang-dev@googlegroups.com</a> mailing list. diff --git a/doc/devel/release.html b/doc/devel/release.html index 84ca622fa..d632200d3 100644 --- a/doc/devel/release.html +++ b/doc/devel/release.html @@ -14,6 +14,78 @@ hg pull hg update release.r<i>NN</i> </pre> +<h2 id="r58">r58 (released 2011/06/29)</h2> + +<p> +The r58 release corresponds to +<code><a href="weekly.html#2011-06-09">weekly.2011-06-09</a></code> +with additional bug fixes. +This section highlights the most significant changes in this release. +For a more detailed summary, see the +<a href="weekly.html#2011-06-09">weekly release notes</a>. +For complete information, see the +<a href="http://code.google.com/p/go/source/list?r=release-branch.r58">Mercurial change list</a>. +</p> + +<h3 id="r58.lang">Language</h3> + +<p> +This release fixes a <a href="http://code.google.com/p/go/source/detail?r=b720749486e1">use of uninitialized memory in programs that misuse <code>goto</code></a>. +</p> + +<h3 id="r58.pkg">Packages</h3> + +<p> +As usual, <a href="/cmd/gofix/">gofix</a> will handle the bulk of the rewrites +necessary for these changes to package APIs. +</p> + +<p> +<a href="/pkg/http/">Package http</a> drops the <code>finalURL</code> return +value from the <a href="/pkg/http/#Client.Get">Client.Get</a> method. The value +is now available via the new <code>Request</code> field on <a +href="/pkg/http/#Response">http.Response</a>. +Most instances of the type map[string][]string in have been +replaced with the new <a href="/pkg/http/#Values">Values</a> type. +</p> + +<p> +<a href="/pkg/exec/">Package exec</a> has been redesigned with a more +convenient and succinct API. +</p> + +<p> +<a href="/pkg/strconv/">Package strconv</a>'s <a href="/pkg/strconv/#Quote">Quote</a> +function now escapes only those Unicode code points not classified as printable +by <a href="/pkg/unicode/#IsPrint">unicode.IsPrint</a>. +Previously Quote would escape all non-ASCII characters. +This also affects the <a href="/pkg/fmt/">fmt</a> package's <code>"%q"</code> +formatting directive. The previous quoting behavior is still available via +strconv's new <a href="/pkg/strconv/#QuoteToASCII">QuoteToASCII</a> function. +</p> + +<p> +<a href="/pkg/os/signal/">Package os/signal</a>'s +<a href="/pkg/os/#Signal">Signal</a> and +<a href="/pkg/os/#UnixSignal">UnixSignal</a> types have been moved to the +<a href="/pkg/os/">os</a> package. +</p> + +<p> +<a href="/pkg/image/draw/">Package image/draw</a> is the new name for +<code>exp/draw</code>. The GUI-related code from <code>exp/draw</code> is now +located in the <a href="/pkg/exp/gui/">exp/gui</a> package. +</p> + +<h3 id="r58.cmd">Tools</h3> + +<p> +<a href="/cmd/goinstall/">Goinstall</a> now observes the GOPATH environment +variable to build and install your own code and external libraries outside of +the Go tree (and avoid writing Makefiles). +</p> + + <h2 id="r57">r57 (released 2011/05/03)</h2> <p> diff --git a/doc/devel/roadmap.html b/doc/devel/roadmap.html index 343203cdb..d3c494715 100644 --- a/doc/devel/roadmap.html +++ b/doc/devel/roadmap.html @@ -43,8 +43,6 @@ Improved garbage collector. <li> Debugger. <li> -App Engine support. -<li> Improved implementation documentation. </ul> @@ -132,4 +130,6 @@ A means of recovering from a panic (recover). Improved CGO including some mechanism for calling back from C to Go. <li> Faster, allocation-light reflection. +<li> +App Engine support. </ul> diff --git a/doc/devel/weekly.html b/doc/devel/weekly.html index 5cb002f73..b576a910d 100644 --- a/doc/devel/weekly.html +++ b/doc/devel/weekly.html @@ -14,11 +14,344 @@ hg pull hg update weekly.<i>YYYY-MM-DD</i> </pre> +<h2 id="2011-06-09">2011-06-09 (<a href="release.html#r58">base for r58</a>)</h2> + +<pre> +This release includes changes to the strconv, http, and exp/draw packages. +Client code that uses the http or exp/draw packages will need to be changed, +and code that uses strconv or fmt's "%q" formatting directive merits checking. + +The strconv package's Quote function now escapes only those Unicode code points +not classified as printable by unicode.IsPrint. Previously Quote would escape +all non-ASCII characters. This also affects the fmt package's "%q" formatting +directive. The previous quoting behavior is still available via strconv's new +QuoteToASCII function. + +Most instances of the type map[string][]string in the http package have been +replaced with the new Values type. The http.Values type has the Get, Set, Add, +and Del helper methods to make working with query parameters and form values +more convenient. + +The exp/draw package has been split into the image/draw and exp/gui packages. + +Other changes: +* 8l, ld: initial adjustments for Plan 9 native compilation of 8l (thanks Lucio De Re). +* arm: floating point improvements (thanks Fan Hongjian). +* big: Improved speed of nat-to-string conversion (thanks Michael T. Jones), + Rat outputs the requested precision from FloatString (thanks Graham Miller), + gobs for big.Rats. +* cgo: support non intel gcc machine flags (thanks Dave Cheney). +* compress/lzw: do not use background goroutines, + reduce decoder buffer size from 3*4096 to 2*4096. +* crypto/twofish: fix Reset index overflow bug. +* crypto: reorg, cleanup and add function for generating CRLs. +* exec: export the underlying *os.Process in Cmd. +* gc: enable building under clang/2.9 (thanks Dave Cheney), + preparatory work toward escape analysis, compact stack frames. +* go/build: new incomplete package for building go programs. +* godefs: do not assume forward type references are enums (thanks Robert Hencke). +* gofix, gofmt: fix diff regression from exec change. +* html: improve attribute parsing, note package status. +* http: don't fail on accept hitting EMFILE, + fix handling of 0-length HTTP requests. +* image/draw: fix clipping bug where sp/mp were not shifted when r.Min was. +* image/gif: fix buglet in graphics extension. +* image/tiff: support for bit depths other than 8 (thanks Benny Siegert). +* ld: fix and simplify ELF symbol generation (thanks Anthony Martin) +* libmach: use the standardized format for designated initializers (thanks Jeff Hodges) +* mail: address list parsing. +* net: add network interface identification API (thanks Mikio Hara), + fix bug in net.Interfaces: handle elastic sdl_data size correctly (thanks Mikio Hara). +* netchan: added drain method to importer (thanks David Jakob Fritz). +* os: add Process.Kill and Process.Signal (thanks Evan Shaw), + fix Getenv for Plan 9 (thanks Yuval Pavel Zholkover). +* runtime: improve memmove by checking memory overlap (thanks Quan Yong Zhai), + support for Linux grsecurity systems (thanks Jonathan Mark). +* spec: handle a corner case for shifts. +* testing: check that tests and benchmarks do not affect GOMAXPROCS (thanks Dmitriy Vyukov). +* unicode: add IsPrint and related properties, general categories. +</pre> + +<h2 id="2011-06-02">2011-06-02</h2> + +<pre> +This release includes changes to the exec package that will require changes +to client code. + +The exec package has been re-designed with a more convenient and succinct API. +This code: + args := []string{"diff", "-u", "file1.txt", "file2.txt"} + p, err := exec.Run("/usr/bin/diff", args, os.Environ(), "", + exec.DevNull, exec.Pipe, exec.DevNull) + if err != nil { + return nil, err + } + var buf bytes.Buffer + io.Copy(&buf, p.Stdout) + w, err := p.Wait(0) + p.Close() + if err != nil { + return nil, err + } + return buf.Bytes(), err +can be rewritten as: + return exec.Command("diff", "-u", "file1.txt", "file2.txt").Output() +See the exec package documentation for the details ("godoc exec"). + +By setting the GOPATH environment variable you can use goinstall to build and +install your own code and external libraries outside of the Go tree (and avoid +writing Makefiles). +See the goinstall command documentation for the details ("godoc goinstall"). + +Other changes: +* 5g: alignment fixes. +* 6l, 8l: fix Mach-O binaries with many dynamic libraries. +* 8l: emit resources (.rsrc) in Windows PE. (thanks Wei Guangjing). +* asn1: fix marshalling of empty optional RawValues (thanks Mikkel Krautz). +* big: make Int and Rat implement fmt.Scanner (thanks Evan Shaw), + ~8x faster number scanning, + remove some unnecessary conversions. +* cgo: restrict #cgo directives to prevent shell expansion (thanks Gustavo Niemeyer), + support pkg-config for flags and libs (thanks Gustavo Niemeyer). +* compress/flate: fix Huffman tree bug, + do not use background goroutines. +* crypto/openpgp: add support for symmetrically encrypting files. +* crypto/tls/generate_cert.go: fix misspelling of O_CREATE. +* dashboard: send notification emails when the build breaks. +* doc: mention go/printer instead of container/vector in effective go, + put Release History link on 'Documentation' page, + put Weekly Snapshot History link on 'Contributing' page. +* encoding/base64: add DecodeString and EncodeToString. +* encoding/binary: add a non-reflect fast path for Read, + add a non-reflect fast path for Write. +* encoding/hex: add hex dumping. +* encoding/line: delete package. Its functionality is now in bufio. +* filepath: Abs must always return a clean path (thanks Gustavo Niemeyer). +* fmt: fix bug in UnreadRune, + make %q work for integers, printing a quoted character literal, + return EOF when out of input in Scan*. +* gc: check parameter declarations in interface fields (thanks Anthony Martin), + disallow ... in type conversions (thanks Anthony Martin), + do not force heap allocation on referencing outer variable in a closure, + fix m[x], _ = y.(T), + implement new shift rules, + patch y.tab.c to fix build when using Bison 2.5, + relax assignability of method receivers (thanks Anthony Martin), + typecheck the whole tree before walking. +* go/scanner: don't allow "0x" and "0X" as integers (thanks Evan Shaw). +* gobuilder: fixes for windows (thanks Alex Brainman). +* godoc: basic setup for running godoc on local app engine emulator, + display advert for the package dashboard on package list page. +* goinstall: fixes for windows (thanks Alex Brainman), + more verbose logging with -v. +* gotest, pkg/exec: use bash to run shell scripts on windows (thanks Alex Brainman). +* http/spdy: redo interfaces, flesh out implementation & frame types (thanks William Chan). +* http: Transport hook to register non-http(s) protocols, + add client+server benchmark, + catch Handler goroutine panics, + fix Set-Cookie date parsing, + have client set Content-Length when possible, + let Transport use a custom net.Dial function, + propagate Set-Cookie in reverse proxy, + ServeFile shouldn't send Content-Length when Content-Encoding is set. +* image: add a SubImage method. +* image/gif: simplify blockReader.Read. +* image/png: fix encoding of images that don't start at (0, 0). +* io, net, http: sendfile support. +* io: add ByteScanner, RuneScanner interfaces. +* ld: add -w to disable dwarf, make errors obviously from dwarf. +* mail: new package. +* mime/multipart: misc code/doc fixes. +* misc/cgo: remove reference to 'destroy' function. +* misc/emacs: don't select the mark after gofmt (thanks Eric Eisner). +* misc/gophertool: Chrome extension to aid in Go development +* misc/vim: limit Fmt command to Go buffers (thanks Yasuhiro Matsumoto). +* net: if we stop polling, remove any pending events for the socket, + update IP multicast socket options (thanks Mikio Hara). +* os: Fix test to work on Solaris, + fix Readdir(0) on EOF, + fix Readdir, Readdirnames (thanks Yuval Pavel Zholkover), + fix os.MkdirAll with backslash path separator (thanks Yasuhiro Matsumoto), + handle OpenFile flag parameter properly on Windows (thanks Alex Brainman). +* path/filepath: remove string constants. +* pkg: spelling tweaks, I-Z (thanks Robert Hencke). +* quietgcc: fix typo, respect $TMPDIR. +* runtime: do not garbage collect windows callbacks (thanks Alex Brainman), + fix mmap error return on linux (thanks Dmitry Chestnykh), + reset GOMAXPROCS during tests, + save cdecl registers in Windows SEH handler (thanks Alexey Borzenkov). +* spec: be precise with the use of the informal ellipsis and the Go token, + clarify rules for shifts. +* strconv: add QuoteRune; analogous to Quote but for runes rather than strings. +* strings: implement UnreadByte, UnreadRune. +* sync: always wake up sleeping goroutines on Cond.Signal (thanks Gustavo Niemeyer). +* sync/atomic: fix check64. +* syscall: add ProcAttr field to pass an unescaped command line on windows (thanks Vincent Vanackere), + add routing messages support for Linux and BSD (thanks Mikio Hara). +* template: fixes and clean-ups (thanks Gustavo Niemeyer). +* time: fix Format bug: midnight/noon are 12AM/PM not 0AM/PM. +* unicode: make the tables smaller. +</pre> + +<h2 id="2011-05-22">2011-05-22</h2> + +<pre> +This release includes changes to the http package that will require changes to +client code. + +The finalURL return value of the Client.Get method has been removed. +This value is now accessible via the new Request field on http.Response. +For example, this code: + + res, finalURL, err := http.Get(...) + +should be rewritten as: + + res, err := http.Get(...) + if err != nil { + // ... + } + finalURL := res.Request.URL.String() + +Uses of http.Get that assign the finalURL value to _ can be rewritten +automatically with gofix. + +This release also includes an optimization to the append function that makes it +between 2 and 5 times faster in typical use cases. + +Other changes: +* 5a, 6a, 8a, cc: remove old environment variables. +* 5c, 5g: fix build with too-smart gcc. +* 5l, 8l: add ELF symbol table to binary. +* 5l: delete pre-ARMv4 instruction implementations, other fixes. +* 6l, 8l: emit windows dwarf sections like other platforms (thanks Alex Brainman). +* 6l: fix emit windows dwarf sections (thanks Wei Guangjing). +* 8g: fix conversion from float to uint64 (thanks Anthony Martin). +* Make.cmd: create TARGDIR if necessary (thanks Gustavo Niemeyer). +* asn1: add big support. +* big: add Int methods to act on numbered bits (thanks Roger Peppe), + better support for string conversions, + support %v and # modifier, better handling of unknown formats. +* cgi: export RequestFromMap (thanks Evan Shaw), + set Request.TLS and Request.RemoteAddr for children. +* cgo: use packed struct to fix Windows behavior. +* codereview: add release branch support, + fetch metadata using JSON API, not XML scraping, + handle 'null as missing field' in rietveld json. +* compress/lzw: silently drop implied codes that are too large. +* compress/zlib: actually use provided dictionary in NewWriterDict +* crypto/openpgp: add key generation support, + change PublicKey.Serialize to include the header. +* crypto/rand: add utility functions for number generation (thanks Anthony Martin). +* crypto/tls: export the verified chains. +* crypto/x509/crl: add package. +* crypto/x509: export raw SubjectPublicKeyInfo, + support DSA public keys in X.509 certs, + support parsing and verifying DSA signatures (thanks Jonathan Allie). +* doc/roadmap: put "App Engine support" under "Done". +* doc: add I/O 2011 talks to talks/, docs.html, and front page. +* effective go: explain about values/pointers in String() example, + update to new Open signature. +* exp/draw: fast paths for drawing a YCbCr or an NRGBA onto an RGBA. +* filepath: make EvalSymlinks work on Windows (thanks Alex Brainman). +* flag: allow distinct sets of flags. +* gc: fix type switch error message for invalid cases (thanks Lorenzo Stoakes), + fix unsafe.Sizeof, + preserve original expression for errors. +* go/ast, go/doc, godoc: consider struct fields and interface methods when filtering ASTs. +* go/ast: consider anonymous fields and set Incomplete bit when filtering ASTs, + properly maintain map of package global imports. +* go/doc, godoc: when filtering for godoc, don't remove elements of a declaration. +* go/parser: accept parenthesized receive operations in select statements, + always introduce an ast.Object when declaring an identifier. +* go/printer, gofmt: fix alignment of "=" in const/var declarations, + fix formatting of expression lists (missing blank). +* go/printer: added simple performance benchmark, + make tests follow syntactic restrictions, + more accurate comment for incomplete structs/interfaces, +* go/token: faster FileSet.Position implementation. +* go/types: type checker API + testing infrastructure. +* godoc: added -index flag to enable/disable search index, + if there is no search box, don't run the respective JS code. +* gofmt: update test.sh (exclude a file w/ incorrect syntax). +* html: parse empty, unquoted, and single-quoted attribute values. +* http/cgi: correctly set request Content-Type (thanks Evan Shaw), + pass down environment variables for IRIX and Solaris. +* http/pprof: fix POST reading bug. +* http/spdy: new incomplete package (thanks Ross Light). +* http: Client.Do should follow redirects for GET and HEAD, + add Header.Write method (thanks Evan Shaw), + add Request.SetBasicAuth method, + add Transport.ProxySelector, + add http.SetCookie(ResponseWriter, *Cookie), + don't Clean query string in relative redirects, + fix FormFile nil pointer dereference on missing multipart form, + fix racy test with a simpler version, + fix two Transport gzip+persist crashes, + include Host header in requests, + make HEAD client request follow redirects (thanks Eivind Uggedal). + update cookie doc to reference new RFC 6265, + write cookies according to RFC 6265 (thanks Christian Himpel). +* image/bmp: implement a BMP decoder. +* image/gif: new package provides a GIF decoder. +* image/jpeg: decode grayscale images, not just color images. + optimizations and tweaks. +* image/png: encode paletted images with alpha channel (thanks Dmitry Chestnykh), + speed up opaque RGBA encoding. +* image/tiff: implement a decoder (thanks Benny Siegert). +* image: add type-specific Set methods and use them when decoding PNG, + make AlphaColor.Set conform to usual signature (thanks Roger Peppe), + png & jpeg encoding benchmarks. +* ld: do not emit reference to dynamic library named "", + fix alignment of rodata section on Plan 9 (thanks Anthony Martin), + make ELF binaries with no shared library dependencies static binaries. +* make.bash: remove old bash version of gotest on Windows (thanks Alex Brainman). +* make: add nuke target for C commands and libs (thanks Anthony Martin). +* mime/multipart: add FileName accessor on Part, + add Writer, + return an error on Reader EOF, not (nil, nil). +* misc/cgo/test: run tests. +* misc/emacs: use UTF-8 when invoking gofmt as a subprocess (thanks Sameer Ajmani). +* misc/vim: new Vim indentation script. +* net, http: add and make use of IP address scope identification API (thanks Mikio Hara). +* net: default to 127.0.0.1, not localhost, in TestICMP, + don't crash on unexpected DNS SRV responses, + enable SO_REUSEPORT on BSD variants (thanks Mikio Hara), + protocol family adaptive address family selection (thanks Mikio Hara), + re-enable wildcard listening (thanks Mikio Hara), + sort records returned by LookupSRV (thanks Gary Burd). +* os: make Readdir & Readdirnames return os.EOF at end, + make Setenv update C environment variables. +* reflect: allow unexported key in Value.MapIndex. +* runtime, sync/atomic: fix arm cas. +* runtime: add newline to "finalizer already set" error (thanks Albert Strasheim), + handle out-of-threads on Linux gracefully (thanks Albert Strasheim), + fix function args not checked warning on ARM (thanks Dave Cheney), + make StackSystem part of StackGuard (thanks Alexey Borzenkov), + maybe fix Windows build broken by cgo setenv CL. +* spec: clarify semantics of integer division, + clarify semantics of range clause, + fix error in production syntax, + narrow syntax for expression and select statements, + newlines cannot be used inside a char or "" string literal, + restricted expressions may still be parenthesized. +* strings: make Reader.Read use copy instead of an explicit loop. +* syscall: add Windows file mapping functions and constants (thanks Evan Shaw), + add IPv6 scope zone ID support (thanks Mikio Hara), + add netlink support for linux/386, linux/amd64, linux/arm (thanks Mikio Hara), + add Sendfile, + adjust freebsd syscalls.master URL properly (thanks Mikio Hara), + change Overlapped.HEvent type, it is a handle (thanks Alex Brainman). +* syslog: fix skipping of net tests (thanks Gustavo Niemeyer). +* template: support string, int and float literals (thanks Gustavo Niemeyer). +* xml: fix reflect error. +</pre> + <h2 id="2011-04-27">2011-04-27 (<a href="release.html#r57">base for r57</a>)</h2> <pre> This release includes revisions to the reflect package to make it more -efficient, after the last weekly’s major API update. If your code uses reflect +efficient, after the last weekly's major API update. If your code uses reflect it may require further changes, not all of which can be made automatically by gofix. For the full details of the change, see http://codereview.appspot.com/4435042 @@ -168,7 +501,7 @@ The os package's Open function has been replaced by three functions: Open(name) // same as old Open(name, O_RDONLY, 0) Create(name) // same as old Open(name, O_RDWR|O_TRUNC|O_CREAT, 0666) -To update your code to use the new APIs, run "gofix path/to/code". Gofix can’t +To update your code to use the new APIs, run "gofix path/to/code". Gofix can't handle all situations perfectly, so read and test the changes it makes before committing them. @@ -298,8 +631,8 @@ produce memory and CPU profiling data when invoked with the -test.memprofile and -test.cpuprofile flags. Run "godoc gotest" for details. We have also introduced a way for tests to run quickly when an exhaustive test -is unnecessary. Gotest’s new -test.short flag in combination with the testing -package’s new Short function allows you to write tests that can be run in +is unnecessary. Gotest's new -test.short flag in combination with the testing +package's new Short function allows you to write tests that can be run in normal or "short" mode; short mode is now used by all.bash to reduce installation time. The Makefiles know about the flag - you can just run "make testshort". diff --git a/doc/docs.html b/doc/docs.html index d4e78446f..9fd3dcebe 100644 --- a/doc/docs.html +++ b/doc/docs.html @@ -70,6 +70,12 @@ An introduction to Go for C++ programmers. <h2 id="tutorials_nonenglish">Non-English Documentation</h2> +<h3 id="docs_be">Belarusian — Беларуская</h3> + +<ul> +<li><a href="http://www.designcontest.com/show/faq-be">faq-be</a> - Frequently Asked Questions.</li> +</ul> + <h3 id="docs_cn">Chinese — 中文</h3> <ul> @@ -90,6 +96,11 @@ An introduction to Go for C++ programmers. <li><a href="http://golang.jp/">golang.jp</a> - Go documentation and news. </ul> +<h3 id="docs_kr">Korean — 한국어</h3> +<ul> +<li><a href="http://code.google.com/p/golang-korea">golang-korea</a> - Go documentation and news. +</ul> + <h3 id="docs_ru">Russian — Русский</h3> <ul> <li><a href="http://golanguage.ru/">golanguage.ru</a> - Go documentation. @@ -119,6 +130,9 @@ The built-in documentation for the Go tools. The official Go Language specification. </p> +<h3 id="release"><a href="devel/release.html">Release History</a></h3> +<p>A summary of the changes between Go releases.</p> + <h3 id="go_mem"><a href="go_mem.html">The Go Memory Model</a></h3> <p> A document that specifies the conditions under which reads of a variable in @@ -128,6 +142,22 @@ same variable in a different goroutine. <h2 id="videos_talks">Videos and Talks</h2> +<h3 id="writing_web_apps"><a href="http://www.youtube.com/watch?v=-i0hat7pdpk">Writing Web Apps in Go</a></h3> +<p> +A talk by Rob Pike and Andrew Gerrand presented at Google I/O 2011. +It walks through the construction and deployment of a simple web application +and unveils the <a href="http://blog.golang.org/2011/05/go-and-google-app-engine.html">Go runtime for App Engine</a>. +See the <a href="/doc/talks/io2011/Writing_Web_Apps_in_Go.pdf">presentation slides</a>. +</p> + +<h3 id="real_world_go"><a href="http://www.youtube.com/watch?v=7QDVRowyUQA">Real World Go</a></h3> +<p> +A talk by Andrew Gerrand presented at Google I/O Bootcamp 2011. +It gives a broad overview of Go's type system and concurrency model +and provides four examples of Go programs that solve real problems. +See the <a href="/doc/talks/io2011/Real_World_Go.pdf">presentation slides</a>. +</p> + <h3 id="go_programming"><a href="http://www.youtube.com/watch?v=jgVhBThJdXc">Go Programming</a></h3> <p> A presentation delivered by Rob Pike and Russ Cox at Google I/O 2010. It diff --git a/doc/effective_go.html b/doc/effective_go.html index a7e6b7bdf..972772d33 100644 --- a/doc/effective_go.html +++ b/doc/effective_go.html @@ -297,9 +297,9 @@ determines just which package is being used. <p> Another convention is that the package name is the base name of its source directory; -the package in <code>src/pkg/container/vector</code> -is imported as <code>"container/vector"</code> but has name <code>vector</code>, -not <code>container_vector</code> and not <code>containerVector</code>. +the package in <code>src/pkg/encoding/base64</code> +is imported as <code>"encoding/base64"</code> but has name <code>base64</code>, +not <code>encoding_base64</code> and not <code>encodingBase64</code>. </p> <p> @@ -504,7 +504,7 @@ the body ends in <code>break</code>, <code>continue</code>, </p> <pre> -f, err := os.Open(name, os.O_RDONLY, 0) +f, err := os.Open(name) if err != nil { return err } @@ -520,7 +520,7 @@ statements, the resulting code needs no <code>else</code> statements. </p> <pre> -f, err := os.Open(name, os.O_RDONLY, 0) +f, err := os.Open(name) if err != nil { return err } @@ -813,7 +813,7 @@ canonical examples are unlocking a mutex or closing a file. <pre> // Contents returns the file's contents as a string. func Contents(filename string) (string, os.Error) { - f, err := os.Open(filename, os.O_RDONLY, 0) + f, err := os.Open(filename) if err != nil { return "", err } @@ -935,13 +935,14 @@ example of its possibilities. <h2 id="data">Data</h2> -<h3 id="allocation_new">Allocation with <code>new()</code></h3> +<h3 id="allocation_new">Allocation with <code>new</code></h3> <p> -Go has two allocation primitives, <code>new()</code> and <code>make()</code>. +Go has two allocation primitives, the built-in functions +<code>new</code> and <code>make</code>. They do different things and apply to different types, which can be confusing, but the rules are simple. -Let's talk about <code>new()</code> first. +Let's talk about <code>new</code> first. It's a built-in function essentially the same as its namesakes in other languages: <code>new(T)</code> allocates zeroed storage for a new item of type <code>T</code> and returns its address, a value of type <code>*T</code>. @@ -950,9 +951,9 @@ In Go terminology, it returns a pointer to a newly allocated zero value of type </p> <p> -Since the memory returned by <code>new()</code> is zeroed, it's helpful to arrange that the +Since the memory returned by <code>new</code> is zeroed, it's helpful to arrange that the zeroed object can be used without further initialization. This means a user of -the data structure can create one with <code>new()</code> and get right to +the data structure can create one with <code>new</code> and get right to work. For example, the documentation for <code>bytes.Buffer</code> states that "the zero value for <code>Buffer</code> is an empty buffer ready to use." @@ -1065,7 +1066,7 @@ s := []string {Enone: "no error", Eio: "Eio", Einval: "invalid argument"} m := map[int]string{Enone: "no error", Eio: "Eio", Einval: "invalid argument"} </pre> -<h3 id="allocation_make">Allocation with <code>make()</code></h3> +<h3 id="allocation_make">Allocation with <code>make</code></h3> <p> Back to allocation. @@ -1099,8 +1100,8 @@ In contrast, <code>new([]int)</code> returns a pointer to a newly allocated, zer structure, that is, a pointer to a <code>nil</code> slice value. <p> -These examples illustrate the difference between <code>new()</code> and -<code>make()</code>. +These examples illustrate the difference between <code>new</code> and +<code>make</code>. </p> <pre> @@ -1116,9 +1117,9 @@ v := make([]int, 100) </pre> <p> -Remember that <code>make()</code> applies only to maps, slices and channels +Remember that <code>make</code> applies only to maps, slices and channels and does not return a pointer. -To obtain an explicit pointer allocate with <code>new()</code>. +To obtain an explicit pointer allocate with <code>new</code>. </p> <h3 id="arrays">Arrays</h3> @@ -1473,7 +1474,7 @@ map[string] int </pre> <p> If you want to control the default format for a custom type, all that's required is to define -a method <code>String() string</code> on the type. +a method with the signature <code>String() string</code> on the type. For our simple type <code>T</code>, that might look like this. </p> <pre> @@ -1489,7 +1490,13 @@ to print in the format 7/-2.35/"abc\tdef" </pre> <p> -Our <code>String()</code> method is able to call <code>Sprintf</code> because the +(If you need to print <em>values</em> of type <code>T</code> as well as pointers to <code>T</code>, +the receiver for <code>String</code> must be of value type; this example used a pointer because +that's more efficient and idiomatic for struct types. +See the section below on <a href="#pointers_vs_values">pointers vs. value receivers</a> for more information.) +</p> +<p> +Our <code>String</code> method is able to call <code>Sprintf</code> because the print routines are fully reentrant and can be used recursively. We can even go one step further and pass a print routine's arguments directly to another such routine. The signature of <code>Printf</code> uses the type <code>...interface{}</code> @@ -1677,19 +1684,20 @@ var ( <h3 id="init">The init function</h3> <p> -Finally, each source file can define its own <code>init()</code> function to -set up whatever state is required. The only restriction is that, although +Finally, each source file can define its own niladic <code>init</code> function to +set up whatever state is required. (Actually each file can have multiple +<code>init</code> functions.) The only restriction is that, although goroutines can be launched during initialization, they will not begin execution until it completes; initialization always runs as a single thread of execution. -And finally means finally: <code>init()</code> is called after all the +And finally means finally: <code>init</code> is called after all the variable declarations in the package have evaluated their initializers, and those are evaluated only after all the imported packages have been initialized. </p> <p> Besides initializations that cannot be expressed as declarations, -a common use of <code>init()</code> functions is to verify or repair +a common use of <code>init</code> functions is to verify or repair correctness of the program state before real execution begins. </p> @@ -1893,7 +1901,7 @@ on every instance of a common method. In such cases, the constructor should return an interface value rather than the implementing type. As an example, in the hash libraries -both <code>crc32.NewIEEE()</code> and <code>adler32.New()</code> +both <code>crc32.NewIEEE</code> and <code>adler32.New</code> return the interface type <code>hash.Hash32</code>. Substituting the CRC-32 algorithm for Adler-32 in a Go program requires only changing the constructor call; @@ -2675,7 +2683,7 @@ field for recoverable failures. <pre> for try := 0; try < 2; try++ { - file, err = os.Open(filename, os.O_RDONLY, 0) + file, err = os.Open(filename) if err == nil { return } diff --git a/doc/frontpage.css b/doc/frontpage.css index bcdca6401..299855ce9 100644 --- a/doc/frontpage.css +++ b/doc/frontpage.css @@ -80,6 +80,9 @@ span.keyword { #content-videos .thumbnail._004 { background: url(/doc/video-004.png); } +#content-videos .thumbnail._005 { + background: url(/doc/video-005.jpg); +} #content-videos a.video { display: inline-block; width: 150px; diff --git a/doc/go_faq.html b/doc/go_faq.html index 5f92b0528..560ab3617 100644 --- a/doc/go_faq.html +++ b/doc/go_faq.html @@ -26,6 +26,7 @@ are not well supported by popular systems languages. <li> The emergence of multicore computers has generated worry and confusion. </ul> +</p> <p> We believe it's worth trying again with a new language, a concurrent, @@ -49,12 +50,14 @@ concurrent execution and communication. By its design, Go proposes an approach for the construction of system software on multicore machines. </ul> +</p> <h3 id="What_is_the_origin_of_the_name"> What is the origin of the name?</h3> <p> “Ogle” would be a good name for a Go debugger. +</p> <h3 id="Whats_the_origin_of_the_mascot"> What's the origin of the mascot?</h3> @@ -83,6 +86,7 @@ http://plan9.bell-labs.com/sys/doc/compiler.html</a> <code>6</code> is the architecture letter for amd64 (or x86-64, if you prefer), while <code>g</code> stands for Go. +</p> <h3 id="history"> What is the history of the project?</h3> @@ -104,7 +108,6 @@ and libraries from prototype to reality. Many others have contributed ideas, discussions, and code. </p> - <h3 id="creating_a_new_language"> Why are you creating a new language?</h3> <p> @@ -117,6 +120,7 @@ language. Programmers who could were choosing ease over safety and efficiency by moving to dynamically typed languages such as Python and JavaScript rather than C++ or, to a lesser extent, Java. </p> + <p> Go is an attempt to combine the ease of programming of an interpreted, dynamically typed @@ -131,7 +135,6 @@ and so on. These cannot be addressed well by libraries or tools; a new language was called for. </p> - <h3 id="ancestors"> What are Go's ancestors?</h3> <p> @@ -147,7 +150,6 @@ about what programmers do and how to make programming, at least the kind of programming we do, more effective, which means more fun. </p> - <h3 id="principles"> What are the guiding principles in the design?</h3> <p> @@ -179,26 +181,17 @@ interfaces represent abstraction; and so on. Orthogonality makes it easier to understand what happens when things combine. </p> - <h2 id="Usage">Usage</h2> -<h3 id="Who_should_use_the_language"> -Who should use the language?</h3> - -<p> -Go is an experiment. We hope adventurous users will give it a try and see -if they enjoy it. Not every programmer -will, but we hope enough will find satisfaction in the approach it -offers to justify further development. - <h3 id="Is_Google_using_go_internally"> Is Google using Go internally?</h3> -<p>Yes. There are now several Go programs deployed in +<p> +Yes. There are now several Go programs deployed in production inside Google. For instance, the server behind <a href="http://golang.org">http://golang.org</a> is a Go program; in fact it's just the <a href="/cmd/godoc"><code>godoc</code></a> document server running in a production configuration. - +</p> <h3 id="Do_Go_programs_link_with_Cpp_programs"> Do Go programs link with C/C++ programs?</h3> @@ -211,11 +204,14 @@ therefore only be linked with C programs using the same convention. There is such a C compiler but no C++ compiler. <code>Gccgo</code> is a GCC front-end that can, with care, be linked with GCC-compiled C or C++ programs. +</p> <p> The <a href="/cmd/cgo/">cgo</a> program provides the mechanism for a “foreign function interface” to allow safe calling of C libraries from Go code. SWIG extends this capability to C++ libraries. +</p> + <h3 id="Does_Go_support_Google_protocol_buffers"> Does Go support Google's protocol buffers?</h3> @@ -226,6 +222,7 @@ It is available at <a href="http://code.google.com/p/goprotobuf/">http://code.google.com/p/goprotobuf/</a> </p> + <h3 id="Can_I_translate_the_Go_home_page"> Can I translate the Go home page into another language?</h3> @@ -277,11 +274,13 @@ such as concurrency and garbage collection. Your favorite feature may be missing because it doesn't fit, because it affects compilation speed or clarity of design, or because it would make the fundamental system model too difficult. +</p> <p> If it bothers you that Go is missing feature <var>X</var>, please forgive us and investigate the features that Go does have. You might find that they compensate in interesting ways for the lack of <var>X</var>. +</p> <h3 id="generics"> Why does Go not have generic types?</h3> @@ -289,6 +288,7 @@ Why does Go not have generic types?</h3> Generics may well be added at some point. We don't feel an urgency for them, although we understand some programmers do. </p> + <p> Generics are convenient but they come at a cost in complexity in the type system and run-time. We haven't yet found a @@ -298,6 +298,7 @@ plus the ability to use the empty interface to construct containers (with explicit unboxing) mean in many cases it is possible to write code that does what generics would enable, if less smoothly. </p> + <p> This remains an open issue. </p> @@ -311,6 +312,7 @@ convoluted code. It also tends to encourage programmers to label too many ordinary errors, such as failing to open a file, as exceptional. </p> + <p> Go takes a different approach. Instead of exceptions, it has a couple of built-in functions to signal and recover from truly exceptional @@ -319,11 +321,11 @@ function's state being torn down after an error, which is sufficient to handle catastrophe but requires no extra control structures and, when used well, can result in clean error-handling code. </p> + <p> See the <a href="http://blog.golang.org/2010/08/defer-panic-and-recover.html">Defer, Panic, and Recover</a> article for details. </p> - <h3 id="assertions"> Why does Go not have assertions?</h3> @@ -336,6 +338,7 @@ Proper error reporting means that errors are direct and to the point, saving the programmer from interpreting a large crash trace. Precise errors are particularly important when the programmer seeing the errors is not familiar with the code. +</p> <p> The same arguments apply to the use of <code>assert()</code> in test programs. Proper @@ -348,17 +351,20 @@ answer for 2 and therefore no more tests were run. The programmer who triggers the test failure may not be familiar with the code that fails. Time invested writing a good error message now pays off later when the test breaks. +</p> <p> In testing, if the amount of extra code required to write good errors seems repetitive and overwhelming, it might work better as a table-driven test instead. Go has excellent support for data structure literals. +</p> <p> We understand that this is a point of contention. There are many things in the Go language and libraries that differ from modern practices, simply because we feel it's sometimes worth trying a different approach. +</p> <h3 id="csp"> Why build concurrency on the ideas of CSP?</h3> @@ -370,6 +376,7 @@ such as mutexes, condition variables, and even memory barriers. Higher-level interfaces enable much simpler code, even if there are still mutexes and such under the covers. </p> + <p> One of the most successful models for providing high-level linguistic support for concurrency comes from Hoare's Communicating Sequential Processes, or CSP. @@ -392,6 +399,7 @@ The result, which we call goroutines, can be very cheap: unless they spend a lot in long-running system calls, they cost little more than the memory for the stack. </p> + <p> To make the stacks small, Go's run-time uses segmented stacks. A newly minted goroutine is given a few kilobytes, which is almost always enough. @@ -420,7 +428,6 @@ as when hosting an untrusted program, the implementation could interlock map access. </p> - <h2 id="types">Types</h2> <h3 id="Is_Go_an_object-oriented_language"> @@ -435,10 +442,12 @@ also ways to embed types in other types to provide something analogous—but not identical—to subclassing. Moreover, methods in Go are more general than in C++ or Java: they can be defined for any sort of data, not just structs. +</p> <p> Also, the lack of type hierarchy makes “objects” in Go feel much more lightweight than in languages such as C++ or Java. +</p> <h3 id="How_do_I_get_dynamic_dispatch_of_methods"> How do I get dynamic dispatch of methods?</h3> @@ -446,6 +455,7 @@ How do I get dynamic dispatch of methods?</h3> <p> The only way to have dynamically dispatched methods is through an interface. Methods on structs or other types are always resolved statically. +</p> <h3 id="inheritance"> Why is there no type inheritance?</h3> @@ -455,6 +465,7 @@ involves too much discussion of the relationships between types, relationships that often could be derived automatically. Go takes a different approach. </p> + <p> Rather than requiring the programmer to declare ahead of time that two types are related, in Go a type automatically satisfies any interface @@ -469,6 +480,7 @@ or for testing—without annotating the original types. Because there are no explicit relationships between types and interfaces, there is no type hierarchy to manage or discuss. </p> + <p> It's possible to use these ideas to construct something analogous to type-safe Unix pipes. For instance, see how <code>fmt.Fprintf</code> @@ -479,6 +491,7 @@ stream ciphers. All these ideas stem from a single interface (<code>io.Writer</code>) representing a single method (<code>Write</code>). And that's only scratching the surface. </p> + <p> It takes some getting used to but this implicit style of type dependency is one of the most exciting things about Go. @@ -503,6 +516,7 @@ but that it could also be confusing and fragile in practice. Matching only by n and requiring consistency in the types was a major simplifying decision in Go's type system. </p> + <p> Regarding operator overloading, it seems more a convenience than an absolute requirement. Again, things are simpler without it. @@ -574,6 +588,23 @@ the interface idea. Sometimes, though, they're necessary to resolve ambiguities among similar interfaces. </p> +<h3 id="convert_slice_of_interface"> +Can I convert a []T to an []interface{}?</h3> + +<p> +Not directly because they do not have the same representation in memory. +It is necessary to copy the elements individually to the destination +slice. This example converts a slice of <code>int</code> to a slice of +<code>interface{}</code>: +</p> + +<pre> +t := []int{1, 2, 3, 4} +s := make([]interface{}, len(t)) +for i, v := range t { + s[i] = v +} +</pre> <h2 id="values">Values</h2> @@ -592,6 +623,7 @@ The definition of constants in Go—arbitrary precision values free of signedness and size annotations—ameliorates matters considerably, though. </p> + <p> A related detail is that, unlike in C, <code>int</code> and <code>int64</code> are distinct types even if <code>int</code> is a 64-bit type. The <code>int</code> @@ -610,7 +642,6 @@ If a specific application can benefit from a custom implementation, it's possibl to write one but it will not be as convenient syntactically; this seems a reasonable tradeoff. </p> - <h3 id="map_keys"> Why don't maps allow structs and arrays as keys?</h3> <p> @@ -637,7 +668,6 @@ language but they have a large effect on usability: Go became a more productive, comfortable language when they were introduced. </p> - <h2 id="Writing_Code">Writing Code</h2> <h3 id="How_are_libraries_documented"> @@ -650,6 +680,7 @@ command line or on the web. An instance is running at <a href="http://golang.org/pkg/">http://golang.org/pkg/</a>. In fact, <code>godoc</code> implements the full site at <a href="http://golang.org/">http://golang.org/</a>. +</p> <h3 id="Is_there_a_Go_programming_style_guide"> Is there a Go programming style guide?</h3> @@ -663,6 +694,7 @@ More directly, the program <code>gofmt</code> is a pretty-printer whose purpose is to enforce layout rules; it replaces the usual compendium of do's and don'ts that allows interpretation. All the Go code in the repository has been run through <code>gofmt</code>. +</p> <h3 id="How_do_I_submit_patches_to_the_Go_libraries"> How do I submit patches to the Go libraries?</h3> @@ -670,12 +702,13 @@ How do I submit patches to the Go libraries?</h3> <p> The library sources are in <code>go/src/pkg</code>. If you want to make a significant change, please discuss on the mailing list before embarking. +</p> <p> See the document <a href="contribute.html">Contributing to the Go project</a> for more information about how to proceed. - +</p> <h2 id="Pointers">Pointers and Allocation</h2> @@ -751,6 +784,7 @@ From a correctness standpoint, you don't need to know. Each variable in Go exists as long as there are references to it. The storage location chosen by the implementation is irrelevant to the semantics of the language. +</p> <p> The storage location does have an effect on writing efficient programs. @@ -759,11 +793,13 @@ local to a function in that function's stack frame. However, if the compiler cannot prove that the variable is not referenced after the function returns, then the compiler must allocate the variable on the garbage-collected heap to avoid dangling pointer errors. +</p> <p> In the current compilers, the analysis is crude: if a variable has its address taken, that variable is allocated on the heap. We are working to improve this analysis so that more data is kept on the stack. +</p> <h2 id="Concurrency">Concurrency</h2> @@ -830,7 +866,6 @@ should recognize such cases and optimize its use of OS threads. For now, <code>GOMAXPROCS</code> should be set on a per-application basis. </p> - <h2 id="Functions_methods">Functions and Methods</h2> <h3 id="different_method_sets"> @@ -994,6 +1029,7 @@ standard GCC back end. <code>Gc</code> is written in C using Although it's a new program, it fits in the Plan 9 C compiler suite (<a href="http://plan9.bell-labs.com/sys/doc/compiler.html">http://plan9.bell-labs.com/sys/doc/compiler.html</a>) and uses a variant of the Plan 9 loader to generate ELF binaries. +</p> <p> We considered writing <code>6g</code>, the original Go compiler, in Go itself but @@ -1003,10 +1039,12 @@ set up a Go environment. <code>Gccgo</code>, which came later, makes it possible consider writing a compiler in Go, which might well happen. (Go would be a fine language in which to implement a compiler; a native lexer and parser are already available in <a href="/pkg/go/"><code>/pkg/go</code></a>.) +</p> <p> We also considered using LLVM for <code>6g</code> but we felt it was too large and slow to meet our performance goals. +</p> <h3 id="How_is_the_run_time_support_implemented"> How is the run-time support implemented?</h3> @@ -1021,6 +1059,7 @@ compiled with a version of the Plan 9 C compiler that supports segmented stacks for goroutines. Work is underway to provide the same stack management in <code>gccgo</code>. +</p> <h3 id="Why_is_my_trivial_program_such_a_large_binary"> Why is my trivial program such a large binary?</h3> @@ -1030,12 +1069,14 @@ The gc tool chain (<code>5l</code>, <code>6l</code>, and <code>8l</code>) only generate statically linked binaries. All Go binaries therefore include the Go run-time, along with the run-time type information necessary to support dynamic type checks, reflection, and even panic-time stack traces. +</p> <p> A trivial C "hello, world" program compiled and linked statically using gcc on Linux is around 750 kB. An equivalent Go program is around 1.1 MB, but that includes more powerful run-time support. We believe that with some effort the size of Go binaries can be reduced. +</p> <h2 id="Performance">Performance</h2> @@ -1070,7 +1111,6 @@ isn't fast enough yet (even if it were, taking care not to generate unnecessary garbage can have a huge effect). </p> - <h2 id="change_from_c">Changes from C</h2> <h3 id="different_syntax"> @@ -1169,6 +1209,7 @@ This works very well in practice but has the effect that it forces a brace style. For instance, the opening brace of a function cannot appear on a line by itself. </p> + <p> Some have argued that the lexer should do lookahead to permit the brace to live on the next line. We disagree. Since Go code is meant @@ -1195,6 +1236,7 @@ implement it with low enough overhead and no significant latency. (The current implementation is a plain mark-and-sweep collector but a replacement is in the works.) </p> + <p> Another point is that a large part of the difficulty of concurrent and multi-threaded programming is memory management; @@ -1205,8 +1247,8 @@ Of course, implementing garbage collection in a concurrent environment is itself a challenge, but meeting it once rather than in every program helps everyone. </p> + <p> Finally, concurrency aside, garbage collection makes interfaces simpler because they don't need to specify how memory is managed across them. </p> - diff --git a/doc/go_mem.html b/doc/go_mem.html index da45a07d7..906b858e8 100644 --- a/doc/go_mem.html +++ b/doc/go_mem.html @@ -1,5 +1,14 @@ <!-- The Go Memory Model --> +<style> +p.rule { + font-style: italic; +} +span.event { + font-style: italic; +} +</style> + <h2>Introduction</h2> <p> @@ -214,6 +223,17 @@ the <code>print</code>. </p> <p class="rule"> +The closing of a channel happens before a receive that returns a zero value +because the channel is closed. +</p> + +<p> +In the previous example, replacing +<code>c <- 0</code> with <code>close(c)</code> +yields a program with the same guaranteed behavior. +</p> + +<p class="rule"> A receive from an unbuffered channel happens before the send on that channel completes. </p> diff --git a/doc/go_spec.html b/doc/go_spec.html index 886f89d12..abf5b8f50 100644 --- a/doc/go_spec.html +++ b/doc/go_spec.html @@ -1,5 +1,5 @@ <!-- title The Go Programming Language Specification --> -<!-- subtitle Version of Apr 22, 2011 --> +<!-- subtitle Version of June 7, 2011 --> <!-- TODO @@ -10,7 +10,6 @@ TODO [ ] clarify what a field name is in struct declarations (struct{T} vs struct {T T} vs struct {t T}) [ ] need explicit language about the result type of operations -[ ] may want to have some examples for the types of shift operations [ ] should string(1<<s) and float32(1<<s) be valid? [ ] should probably write something about evaluation order of statements even though obvious @@ -46,10 +45,10 @@ The syntax is specified using Extended Backus-Naur Form (EBNF): </p> <pre class="grammar"> -Production = production_name "=" Expression "." . +Production = production_name "=" [ Expression ] "." . Expression = Alternative { "|" Alternative } . Alternative = Term { Term } . -Term = production_name | token [ "..." token ] | Group | Option | Repetition . +Term = production_name | token [ "…" token ] | Group | Option | Repetition . Group = "(" Expression ")" . Option = "[" Expression "]" . Repetition = "{" Expression "}" . @@ -73,8 +72,12 @@ double quotes <code>""</code> or back quotes <code>``</code>. </p> <p> -The form <code>a ... b</code> represents the set of characters from -<code>a</code> through <code>b</code> as alternatives. +The form <code>a … b</code> represents the set of characters from +<code>a</code> through <code>b</code> as alternatives. The horizontal +ellipis … is also used elsewhere in the spec to informally denote various +enumerations or code snippets that are not further specified. The character … +(as opposed to the three characters <code>...</code>) is not a token of the Go +language. </p> <h2 id="Source_code_representation">Source code representation</h2> @@ -102,7 +105,8 @@ compiler may disallow the NUL character (U+0000) in the source text. The following terms are used to denote specific Unicode character classes: </p> <pre class="ebnf"> -unicode_char = /* an arbitrary Unicode code point */ . +newline = /* the Unicode code point U+000A */ . +unicode_char = /* an arbitrary Unicode code point except newline */ . unicode_letter = /* a Unicode code point classified as "Letter" */ . unicode_digit = /* a Unicode code point classified as "Decimal Digit" */ . </pre> @@ -122,9 +126,9 @@ The underscore character <code>_</code> (U+005F) is considered a letter. </p> <pre class="ebnf"> letter = unicode_letter | "_" . -decimal_digit = "0" ... "9" . -octal_digit = "0" ... "7" . -hex_digit = "0" ... "9" | "A" ... "F" | "a" ... "f" . +decimal_digit = "0" … "9" . +octal_digit = "0" … "7" . +hex_digit = "0" … "9" | "A" … "F" | "a" … "f" . </pre> <h2 id="Lexical_elements">Lexical elements</h2> @@ -286,7 +290,7 @@ An optional prefix sets a non-decimal base: <code>0</code> for octal, <code>0x</ </p> <pre class="ebnf"> int_lit = decimal_lit | octal_lit | hex_lit . -decimal_lit = ( "1" ... "9" ) { decimal_digit } . +decimal_lit = ( "1" … "9" ) { decimal_digit } . octal_lit = "0" { octal_digit } . hex_lit = "0" ( "x" | "X" ) hex_digit { hex_digit } . </pre> @@ -471,7 +475,7 @@ U+00FF. <pre class="ebnf"> string_lit = raw_string_lit | interpreted_string_lit . -raw_string_lit = "`" { unicode_char } "`" . +raw_string_lit = "`" { unicode_char | newline } "`" . interpreted_string_lit = `"` { unicode_value | byte_value } `"` . </pre> @@ -821,7 +825,7 @@ make([]T, length, capacity) </pre> <p> -The <code>make()</code> call allocates a new, hidden array to which the returned +A call to <code>make</code> allocates a new, hidden array to which the returned slice value refers. That is, executing </p> @@ -1053,9 +1057,9 @@ have the method set </p> <pre> -func (p T) Read(b Buffer) bool { return ... } -func (p T) Write(b Buffer) bool { return ... } -func (p T) Close() { ... } +func (p T) Read(b Buffer) bool { return … } +func (p T) Write(b Buffer) bool { return … } +func (p T) Close() { … } </pre> <p> @@ -1093,8 +1097,8 @@ If <code>S1</code> and <code>S2</code> also implement </p> <pre> -func (p T) Lock() { ... } -func (p T) Unlock() { ... } +func (p T) Lock() { … } +func (p T) Unlock() { … } </pre> <p> @@ -1173,8 +1177,9 @@ make(map[string] int, 100) <p> The initial capacity does not bound its size: maps grow to accommodate the number of items -stored in them. -</p> +stored in them, with the exception of <code>nil</code> maps. +A <code>nil</code> map is equivalent to an empty map except that no elements +may be added. <h3 id="Channel_types">Channel types</h3> @@ -1233,6 +1238,7 @@ succeed without blocking if the buffer is not full (sends) or not empty (receive and elements are received in the order they are sent. If the capacity is zero or absent, the communication succeeds only when both a sender and receiver are ready. +A <code>nil</code> channel is never ready for communication. </p> <p> @@ -1693,7 +1699,7 @@ of an interface type or of elements of a composite type remains unchanged: </p> <pre> -// A Mutex is a data type with two methods Lock and Unlock. +// A Mutex is a data type with two methods, Lock and Unlock. type Mutex struct { /* Mutex fields */ } func (m *Mutex) Lock() { /* Lock implementation */ } func (m *Mutex) Unlock() { /* Unlock implementation */ } @@ -1967,11 +1973,11 @@ package, which means that it must begin with a Unicode upper case letter. math.Sin </pre> -<!--- +<!-- <p> <span class="alert">TODO: Unify this section with Selectors - it's the same syntax.</span> </p> ----> +--> <h3 id="Composite_literals">Composite literals</h3> @@ -2096,7 +2102,7 @@ element index plus one. A slice literal has the form </p> <pre> -[]T{x1, x2, ... xn} +[]T{x1, x2, … xn} </pre> <p> @@ -2104,7 +2110,7 @@ and is a shortcut for a slice operation applied to an array literal: </p> <pre> -[n]T{x1, x2, ... xn}[0 : n] +[n]T{x1, x2, … xn}[0 : n] </pre> <p> @@ -2130,8 +2136,8 @@ parentheses. </p> <pre> -if x == (T{a,b,c}[i]) { ... } -if (x == T{a,b,c}[i]) { ... } +if x == (T{a,b,c}[i]) { … } +if (x == T{a,b,c}[i]) { … } </pre> <p> @@ -2335,11 +2341,11 @@ p.M0 // ((*p).T0).M0 </pre> -<!--- +<!-- <span class="alert"> TODO: Specify what happens to receivers. </span> ----> +--> <h3 id="Indexes">Indexes</h3> @@ -2368,7 +2374,7 @@ or for <code>a</code> of type <code>S</code> where <code>S</code> is a <a href=" <li><code>x</code> must be an integer value and <code>0 <= x < len(a)</code></li> <li><code>a[x]</code> is the array element at index <code>x</code> and the type of <code>a[x]</code> is the element type of <code>A</code></li> - <li>if the index <code>x</code> is out of range, + <li>if <code>a</code> is <code>nil</code> or if the index <code>x</code> is out of range, a <a href="#Run_time_panics">run-time panic</a> occurs</li> </ul> @@ -2396,7 +2402,7 @@ where <code>M</code> is a <a href="#Map_types">map type</a>: <li>if the map contains an entry with key <code>x</code>, <code>a[x]</code> is the map value with key <code>x</code> and the type of <code>a[x]</code> is the value type of <code>M</code></li> - <li>if the map does not contain such an entry, + <li>if the map is <code>nil</code> or does not contain such an entry, <code>a[x]</code> is the <a href="#The_zero_value">zero value</a> for the value type of <code>M</code></li> </ul> @@ -2425,7 +2431,7 @@ where the result of the index expression is a pair of values with types </p> <p> -Similarly, if an assignment to a map has the special form +Similarly, if an assignment to a map element has the special form </p> <pre> @@ -2439,6 +2445,11 @@ the entry for key <code>x</code> is deleted from the map; if a regular assignment to an element of the map. </p> +<p> +Assigning to an element of a <code>nil</code> map causes a +<a href="#Run_time_panics">run-time panic</a>. +</p> + <h3 id="Slices">Slices</h3> @@ -2559,11 +2570,11 @@ Given an expression <code>f</code> of function type </p> <pre> -f(a1, a2, ... an) +f(a1, a2, … an) </pre> <p> -calls <code>f</code> with arguments <code>a1, a2, ... an</code>. +calls <code>f</code> with arguments <code>a1, a2, … an</code>. Except for one special case, arguments must be single-valued expressions <a href="#Assignability">assignable</a> to the parameter types of <code>F</code> and are evaluated before the function is called. @@ -2642,7 +2653,7 @@ arguments bound to the final parameter and may differ for each call site. Given the function and call </p> <pre> -func Greeting(prefix string, who ... string) +func Greeting(prefix string, who ...string) Greeting("hello:", "Joe", "Anna", "Eileen") </pre> @@ -2693,42 +2704,39 @@ unary_op = "+" | "-" | "!" | "^" | "*" | "&" | "<-" . <p> Comparisons are discussed <a href="#Comparison_operators">elsewhere</a>. For other binary operators, the operand types must be <a href="#Type_identity">identical</a> -unless the operation involves channels, shifts, or untyped <a href="#Constants">constants</a>. +unless the operation involves shifts or untyped <a href="#Constants">constants</a>. For operations involving constants only, see the section on <a href="#Constant_expressions">constant expressions</a>. </p> <p> -In a channel send, the first operand is always a channel and the second -must be a value <a href="#Assignability">assignable</a> -to the channel's element type. -</p> - -<p> -Except for shift operations, -if one operand is an untyped <a href="#Constants">constant</a> +Except for shift operations, if one operand is an untyped <a href="#Constants">constant</a> and the other operand is not, the constant is <a href="#Conversions">converted</a> to the type of the other operand. </p> <p> -The right operand in a shift operation must have unsigned integer type +The right operand in a shift expression must have unsigned integer type or be an untyped constant that can be converted to unsigned integer type. -</p> - -<p> -If the left operand of a non-constant shift operation is an untyped constant, -the type of constant is what it would be if the shift operation were replaced by -the left operand alone. +If the left operand of a non-constant shift expression is an untyped constant, +the type of the constant is what it would be if the shift expression were +replaced by its left operand alone; the type is <code>int</code> if it cannot +be determined from the context (for instance, if the shift expression is an +operand in a comparison against an untyped constant). </p> <pre> var s uint = 33 -var i = 1<<s // 1 has type int -var j = int32(1<<s) // 1 has type int32; j == 0 -var u = uint64(1<<s) // 1 has type uint64; u == 1<<33 -var f = float32(1<<s) // illegal: 1 has type float32, cannot shift -var g = float32(1<<33) // legal; 1<<33 is a constant shift operation; g == 1<<33 +var i = 1<<s // 1 has type int +var j int32 = 1<<s // 1 has type int32; j == 0 +var k = uint64(1<<s) // 1 has type uint64; k == 1<<33 +var m int = 1.0<<s // legal: 1.0 has type int +var n = 1.0<<s != 0 // legal: 1.0 has type int; n == false if ints are 32bits in size +var o = 1<<s == 2<<s // legal: 1 and 2 have type int; o == true if ints are 32bits in size +var p = 1<<s == 1<<33 // illegal if ints are 32bits in size: 1 has type int, but 1<<33 overflows int +var u = 1.0<<s // illegal: 1.0 has type float64, cannot shift +var v float32 = 1<<s // illegal: 1 has type float32, cannot shift +var w int64 = 1.0<<33 // legal: 1.0<<33 is a constant shift expression </pre> <h3 id="Operator_precedence">Operator precedence</h3> @@ -2808,15 +2816,18 @@ s += " and good bye" String addition creates a new string by concatenating the operands. </p> <p> -For integer values, <code>/</code> and <code>%</code> satisfy the following relationship: +For two integer values <code>x</code> and <code>y</code>, the integer quotient +<code>q = x / y</code> and remainder <code>r = x % y</code> satisfy the following +relationships: </p> <pre> -(a / b) * b + a % b == a +x = q*y + r and |r| < |y| </pre> <p> -with <code>(a / b)</code> truncated towards zero. +with <code>x / y</code> truncated towards zero +(<a href="http://en.wikipedia.org/wiki/Modulo_operation">"truncated division"</a>). </p> <pre> @@ -2828,6 +2839,20 @@ with <code>(a / b)</code> truncated towards zero. </pre> <p> +As an exception to this rule, if the dividend <code>x</code> is the most +negative value for the int type of <code>x</code>, the quotient +<code>q = x / -1</code> is equal to <code>x</code> (and <code>r = 0</code>). +</p> + +<pre> + x, q +int8 -128 +int16 -32768 +int32 -2147483648 +int64 -9223372036854775808 +</pre> + +<p> If the divisor is zero, a <a href="#Run_time_panics">run-time panic</a> occurs. If the dividend is positive and the divisor is a constant power of 2, the division may be replaced by a right shift, and computing the remainder may @@ -2843,8 +2868,8 @@ be replaced by a bitwise "and" operation: <p> The shift operators shift the left operand by the shift count specified by the right operand. They implement arithmetic shifts if the left operand is a signed -integer and logical shifts if it is an unsigned integer. The shift count must -be an unsigned integer. There is no upper limit on the shift count. Shifts behave +integer and logical shifts if it is an unsigned integer. +There is no upper limit on the shift count. Shifts behave as if the left operand is shifted <code>n</code> times by 1 for a shift count of <code>n</code>. As a result, <code>x << 1</code> is the same as <code>x*2</code> @@ -3025,6 +3050,7 @@ For an operand <code>ch</code> of <a href="#Channel_types">channel type</a>, the value of the receive operation <code><-ch</code> is the value received from the channel <code>ch</code>. The type of the value is the element type of the channel. The expression blocks until a value is available. +Receiving from a <code>nil</code> channel blocks forever. </p> <pre> @@ -3052,17 +3078,12 @@ or is a <a href="#The_zero_value">zero value</a> returned because the channel is closed and empty (<code>false</code>). </p> -<p> -Receiving from a <code>nil</code> channel causes a -<a href="#Run_time_panics">run-time panic</a>. -</p> - -<!--- +<!-- <p> <span class="alert">TODO: Probably in a separate section, communication semantics need to be presented regarding send, receive, select, and goroutines.</span> </p> ----> +--> <h3 id="Method_expressions">Method expressions</h3> @@ -3360,21 +3381,35 @@ respectively. Except for shift operations, if the operands of a binary operation are an untyped integer constant and an untyped floating-point constant, the integer constant is converted to an untyped floating-point constant (relevant for <code>/</code> and <code>%</code>). -Similarly, -untyped integer or floating-point constants may be used as operands +Similarly, untyped integer or floating-point constants may be used as operands wherever it is legal to use an operand of complex type; the integer or floating point constant is converted to a complex constant with a zero imaginary part. </p> <p> -Applying an operator to untyped constants results in an untyped +A constant <a href="#Comparison_operators">comparison</a> always yields +a constant of type <code>bool</code>. If the left operand of a constant +<a href="#Operators">shift expression</a> is an untyped constant, the +result is an integer constant; otherwise it is a constant of the same +type as the left operand, which must be of integer type +(§<a href="#Arithmetic_operators">Arithmetic operators</a>). +Applying all other operators to untyped constants results in an untyped constant of the same kind (that is, a boolean, integer, floating-point, -complex, or string constant), except for -<a href="#Comparison_operators">comparison operators</a>, which result in -a constant of type <code>bool</code>. +complex, or string constant). </p> +<pre> +const a = 2 + 3.0 // a == 5.0 (floating-point constant) +const b = 15 / 4 // b == 3 (integer constant) +const c = 15 / 4.0 // c == 3.75 (floating-point constant) +const d = 1 << 3.0 // d == 8 (integer constant) +const e = 1.0 << 3 // e == 8 (integer constant) +const f = int32(1) << 33 // f == 0 (type int32) +const g = float64(2) >> 1 // illegal (float64(2) is a typed floating-point constant) +const h = "foo" > "bar" // h == true (type bool) +</pre> + <p> Imaginary literals are untyped complex constants (with zero real part) and may be combined in binary @@ -3431,7 +3466,7 @@ int8(^1) // same as int8(-2) ^int8(1) // same as -1 ^ int8(1) = -2 </pre> -<!--- +<!-- <p> <span class="alert"> TODO: perhaps ^ should be disallowed on non-uints instead of assuming twos complement. @@ -3439,7 +3474,7 @@ Also it may be possible to make typed constants more like variables, at the cost overflow etc. errors being caught. </span> </p> ----> +--> <h3 id="Order_of_evaluation">Order of evaluation</h3> @@ -3522,10 +3557,9 @@ Error: log.Panic("error encountered") <p> Function calls, method calls, and receive operations -can appear in statement context. +can appear in statement context. Such statements may be parenthesized. </p> - <pre class="ebnf"> ExpressionStmt = Expression . </pre> @@ -3534,6 +3568,7 @@ ExpressionStmt = Expression . h(x+y) f.Close() <-ch +(<-ch) </pre> @@ -3557,17 +3592,13 @@ begins. Communication blocks until the send can proceed, at which point the value is transmitted on the channel. A send on an unbuffered channel can proceed if a receiver is ready. A send on a buffered channel can proceed if there is room in the buffer. +A send on a <code>nil</code> channel blocks forever. </p> <pre> ch <- 3 </pre> -<p> -Sending to a <code>nil</code> channel causes a -<a href="#Run_time_panics">run-time panic</a>. -</p> - <h3 id="IncDec_statements">IncDec statements</h3> @@ -3604,15 +3635,15 @@ assign_op = [ add_op | mul_op ] "=" . <p> Each left-hand side operand must be <a href="#Address_operators">addressable</a>, -a map index expression, -or the <a href="#Blank_identifier">blank identifier</a>. +a map index expression, or the <a href="#Blank_identifier">blank identifier</a>. +Operands may be parenthesized. </p> <pre> x = 1 *p = f() a[i] = 23 -k = <-ch +(k) = <-ch // same as: k = <-ch </pre> <p> @@ -3966,10 +3997,14 @@ As with an assignment, the operands on the left must be <a href="#Address_operators">addressable</a> or map index expressions; they denote the iteration variables. If the range expression is a channel, only one iteration variable is permitted, otherwise there may be one or two. +If the second iteration variable is the <a href="#Blank_identifier">blank identifier</a>, +the range clause is equivalent to the same clause with only the first variable present. </p> <p> -The range expression is evaluated once before beginning the loop. +The range expression is evaluated once before beginning the loop +except if the expression is an array, in which case, depending on +the expression, it might not be evaluated (see below). Function calls on the left are evaluated once per iteration. For each iteration, iteration values are produced as follows: </p> @@ -3985,8 +4020,11 @@ channel c chan E element e E <ol> <li> -For an array or slice value, the index iteration values are produced in -increasing order, starting at element index 0. +For an array, pointer to array, or slice value <code>a</code>, the index iteration +values are produced in increasing order, starting at element index 0. As a special +case, if only the first iteration variable is present, the range loop produces +iteration values from 0 up to <code>len(a)</code> and does not index into the array +or slice itself. For a <code>nil</code> slice, the number of iterations is 0. </li> <li> @@ -4005,13 +4043,14 @@ The iteration order over maps is not specified. If map entries that have not yet been reached are deleted during iteration, the corresponding iteration values will not be produced. If map entries are inserted during iteration, the behavior is implementation-dependent, but the -iteration values for each entry will be produced at most once. +iteration values for each entry will be produced at most once. If the map +is <code>nil</code>, the number of iterations is 0. </li> <li> For channels, the iteration values produced are the successive values sent on -the channel until the channel is closed -(§<a href="#Close"><code>close</code>). +the channel until the channel is <a href="#Close">closed</a>. If the channel +is <code>nil</code>, the range expression blocks forever. </li> </ol> @@ -4030,9 +4069,17 @@ after execution their values will be those of the last iteration. </p> <pre> +var testdata *struct { + a *[7]int +} +for i, _ := range testdata.a { + // testdata.a is never evaluated; len(testdata.a) is constant + // i ranges from 0 to 6 + f(i) +} + var a [10]string m := map[string]int{"mon":0, "tue":1, "wed":2, "thu":3, "fri":4, "sat":5, "sun":6} - for i, s := range a { // type of i is int // type of s is string @@ -4047,6 +4094,11 @@ for key, val = range m { } // key == last map key encountered in iteration // val == map[key] + +var ch chan Work = producer() +for w := range ch { + doWork(w) +} </pre> @@ -4131,7 +4183,7 @@ case i1 = <-c1: print("received ", i1, " from c1\n") case c2 <- i2: print("sent ", i2, " to c2\n") -case i3, ok := <-c3: +case i3, ok := (<-c3): // same as: i3, ok := <-c3 if ok { print("received ", i3, " from c3\n") } else { @@ -4229,7 +4281,7 @@ func (devnull) Write(p []byte) (n int, _ os.Error) { Regardless of how they are declared, all the result values are initialized to the zero values for their type (§<a href="#The_zero_value">The zero value</a>) upon entry to the function. </p> -<!--- +<!-- <p> <span class="alert"> TODO: Define when return is required.<br /> @@ -4237,7 +4289,7 @@ TODO: Language about result parameters needs to go into a section on function/method invocation<br /> </span> </p> ----> +--> <h3 id="Break_statements">Break statements</h3> @@ -4314,9 +4366,9 @@ L: <p> is erroneous because the jump to label <code>L</code> skips the creation of <code>v</code>. -<!--- +<!-- (<span class="alert">TODO: Eliminate in favor of used and not set errors?</span>) ----> +--> </p> <h3 id="Fallthrough_statements">Fallthrough statements</h3> @@ -4421,17 +4473,17 @@ The implementation guarantees that the result always fits into an <code>int</cod </p> <pre class="grammar"> -Call Argument type Result +Call Argument type Result -len(s) string type string length in bytes - [n]T, *[n]T array length (== n) - []T slice length - map[K]T map length (number of defined keys) - chan T number of elements queued in channel buffer +len(s) string type string length in bytes + [n]T, *[n]T array length (== n) + []T slice length + map[K]T map length (number of defined keys) + chan T number of elements queued in channel buffer -cap(s) [n]T, *[n]T array length (== n) - []T slice capacity - chan T channel buffer capacity +cap(s) [n]T, *[n]T array length (== n) + []T slice capacity + chan T channel buffer capacity </pre> <p> @@ -4449,20 +4501,17 @@ The length and capacity of a <code>nil</code> slice, map, or channel are 0. </p> <p> -The expression -<code>len(s)</code> is a -<a href="#Constants">constant</a> if <code>s</code> is a string constant. -The expressions -<code>len(s)</code> and -<code>cap(s)</code> are -constants if <code>s</code> is an (optionally parenthesized) -identifier or -<a href="#Qualified_identifiers">qualified identifier</a> -denoting an array or pointer to array. -Otherwise invocations of <code>len</code> and <code>cap</code> are not -constant. +The expression <code>len(s)</code> is <a href="#Constants">constant</a> if +<code>s</code> is a string constant. The expressions <code>len(s)</code> and +<code>cap(s)</code> are constants if the type of <code>s</code> is an array +or pointer to an array and the expression <code>s</code> does not contain +<a href="#Receive_operator">channel receives</a> or +<a href="#Calls">function calls</a>; in this case <code>s</code> is not evaluated. +Otherwise, invocations of <code>len</code> and <code>cap</code> are not +constant and <code>s</code> is evaluated. </p> + <h3 id="Allocation">Allocation</h3> <p> @@ -4575,7 +4624,7 @@ a source <code>src</code> to a destination <code>dst</code> and returns the number of elements copied. Source and destination may overlap. Both arguments must have <a href="#Type_identity">identical</a> element type <code>T</code> and must be <a href="#Assignability">assignable</a> to a slice of type <code>[]T</code>. -The number of arguments copied is the minimum of +The number of elements copied is the minimum of <code>len(src)</code> and <code>len(dst)</code>. As a special case, <code>copy</code> also accepts a destination argument assignable to type <code>[]byte</code> with a source argument of a string type. @@ -4850,7 +4899,7 @@ package main import "fmt" -// Send the sequence 2, 3, 4, ... to channel 'ch'. +// Send the sequence 2, 3, 4, … to channel 'ch'. func generate(ch chan<- int) { for i := 2; ; i++ { ch <- i // Send 'i' to channel 'ch'. @@ -4890,7 +4939,7 @@ func main() { <h3 id="The_zero_value">The zero value</h3> <p> When memory is allocated to store a value, either through a declaration -or <code>make()</code> or <code>new()</code> call, +or a call of <code>make</code> or <code>new</code>, and no explicit initialization is provided, the memory is given a default initialization. Each element of such a value is set to the <i>zero value</i> for its type: <code>false</code> for booleans, @@ -4948,7 +4997,7 @@ func init() <p> defined in its source. A package may contain multiple -<code>init()</code> functions, even +<code>init</code> functions, even within a single source file; they execute in unspecified order. </p> @@ -4978,8 +5027,8 @@ program is complete. Therefore, all initialization code is run in a single goroutine. </p> <p> -An <code>init()</code> function cannot be referred to from anywhere -in a program. In particular, <code>init()</code> cannot be called explicitly, +An <code>init</code> function cannot be referred to from anywhere +in a program. In particular, <code>init</code> cannot be called explicitly, nor can a pointer to <code>init</code> be assigned to a function variable. </p> <p> @@ -5001,7 +5050,7 @@ arguments and returns no value. </p> <pre> -func main() { ... } +func main() { … } </pre> <p> @@ -5151,5 +5200,8 @@ The following minimal alignment properties are guaranteed: <h2 id="Implementation_differences"><span class="alert">Implementation differences - TODO</span></h2> <ul> - <li><span class="alert">Implementation does not honor the restriction on goto statements and targets (no intervening declarations).</span></li> + <li><span class="alert">The restriction on <code>goto</code> statements and targets (no intervening declarations) is not honored.</span></li> + <li><span class="alert"><code>len(a)</code> is only a constant if <code>a</code> is a (qualified) identifier denoting an array or pointer to an array.</span></li> + <li><span class="alert"><code>nil</code> maps are not treated like empty maps.</span></li> + <li><span class="alert">Trying to send/receive from a <code>nil</code> channel causes a run-time panic.</span></li> </ul> diff --git a/doc/go_tutorial.html b/doc/go_tutorial.html index 4470f2748..aa85134b3 100644 --- a/doc/go_tutorial.html +++ b/doc/go_tutorial.html @@ -28,9 +28,9 @@ Let's start in the usual way: <p> <pre> <!-- progs/helloworld.go /package/ END --> 05 package main -<p> + 07 import fmt "fmt" // Package implementing formatted I/O. -<p> + 09 func main() { 10 fmt.Printf("Hello, world; or Καλημέρα κόσμε; or こんにちは 世界\n") 11 } @@ -118,19 +118,19 @@ Next up, here's a version of the Unix utility <code>echo(1)</code>: <p> <pre> <!-- progs/echo.go /package/ END --> 05 package main -<p> + 07 import ( 08 "os" 09 "flag" // command line option parser 10 ) -<p> + 12 var omitNewline = flag.Bool("n", false, "don't print final newline") -<p> + 14 const ( 15 Space = " " 16 Newline = "\n" 17 ) -<p> + 19 func main() { 20 flag.Parse() // Scans the arg list and sets up flags 21 var s string = "" @@ -340,7 +340,7 @@ Using slices one can write this function (from <code>sum.go</code>): 15 } </pre> <p> -Note how the return type (<code>int</code>) is defined for <code>sum()</code> by stating it +Note how the return type (<code>int</code>) is defined for <code>sum</code> by stating it after the parameter list. <p> To call the function, we slice the array. This intricate call (we'll show @@ -372,7 +372,7 @@ There are also maps, which you can initialize like this: m := map[string]int{"one":1 , "two":2} </pre> <p> -The built-in function <code>len()</code>, which returns number of elements, +The built-in function <code>len</code>, which returns number of elements, makes its first appearance in <code>sum</code>. It works on strings, arrays, slices, maps, and channels. <p> @@ -400,7 +400,7 @@ for more examples of its use. Most types in Go are values. If you have an <code>int</code> or a <code>struct</code> or an array, assignment copies the contents of the object. -To allocate a new variable, use <code>new()</code>, which +To allocate a new variable, use the built-in function <code>new</code>, which returns a pointer to the allocated storage. <p> <pre> @@ -417,7 +417,7 @@ or the more idiomatic Some types—maps, slices, and channels (see below)—have reference semantics. If you're holding a slice or a map and you modify its contents, other variables referencing the same underlying data will see the modification. For these three -types you want to use the built-in function <code>make()</code>: +types you want to use the built-in function <code>make</code>: <p> <pre> m := make(map[string]int) @@ -431,11 +431,11 @@ If you just declare the map, as in </pre> <p> it creates a <code>nil</code> reference that cannot hold anything. To use the map, -you must first initialize the reference using <code>make()</code> or by assignment from an +you must first initialize the reference using <code>make</code> or by assignment from an existing map. <p> Note that <code>new(T)</code> returns type <code>*T</code> while <code>make(T)</code> returns type -<code>T</code>. If you (mistakenly) allocate a reference object with <code>new()</code>, +<code>T</code>. If you (mistakenly) allocate a reference object with <code>new</code> rather than <code>make</code>, you receive a pointer to a nil reference, equivalent to declaring an uninitialized variable and taking its address. <p> @@ -478,12 +478,12 @@ open/close/read/write interface. Here's the start of <code>file.go</code>: <p> <pre> <!-- progs/file.go /package/ /^}/ --> 05 package file -<p> + 07 import ( 08 "os" 09 "syscall" 10 ) -<p> + 12 type File struct { 13 fd int // file descriptor number 14 name string // file name at Open time @@ -600,7 +600,7 @@ the tricky standard arguments to open and, especially, to create a file: 41 O_CREATE = syscall.O_CREAT 42 O_TRUNC = syscall.O_TRUNC 43 ) -<p> + 45 func Open(name string) (file *File, err os.Error) { 46 return OpenFile(name, O_RDONLY, 0) 47 } @@ -631,7 +631,7 @@ each of which declares a receiver variable <code>file</code>. 61 } 62 return nil 63 } -<p> + 65 func (file *File) Read(b []byte) (ret int, err os.Error) { 66 if file == nil { 67 return -1, os.EINVAL @@ -642,7 +642,7 @@ each of which declares a receiver variable <code>file</code>. 72 } 73 return int(r), err 74 } -<p> + 76 func (file *File) Write(b []byte) (ret int, err os.Error) { 77 if file == nil { 78 return -1, os.EINVAL @@ -653,7 +653,7 @@ each of which declares a receiver variable <code>file</code>. 83 } 84 return int(r), err 85 } -<p> + 87 func (file *File) String() string { 88 return file.name 89 } @@ -676,13 +676,13 @@ We can now use our new package: <p> <pre> <!-- progs/helloworld3.go /package/ END --> 05 package main -<p> + 07 import ( 08 "./file" 09 "fmt" 10 "os" 11 ) -<p> + 13 func main() { 14 hello := []byte("hello, world\n") 15 file.Stdout.Write(hello) @@ -719,14 +719,14 @@ Building on the <code>file</code> package, here's a simple version of the Unix u <p> <pre> <!-- progs/cat.go /package/ END --> 05 package main -<p> + 07 import ( 08 "./file" 09 "flag" 10 "fmt" 11 "os" 12 ) -<p> + 14 func cat(f *file.File) { 15 const NBUF = 512 16 var buf [NBUF]byte @@ -745,7 +745,7 @@ Building on the <code>file</code> package, here's a simple version of the Unix u 29 } 30 } 31 } -<p> + 33 func main() { 34 flag.Parse() // Scans the arg list and sets up flags 35 if flag.NArg() == 0 { @@ -766,7 +766,7 @@ Building on the <code>file</code> package, here's a simple version of the Unix u By now this should be easy to follow, but the <code>switch</code> statement introduces some new features. Like a <code>for</code> loop, an <code>if</code> or <code>switch</code> can include an initialization statement. The <code>switch</code> on line 18 uses one to create variables -<code>nr</code> and <code>er</code> to hold the return values from <code>f.Read()</code>. (The <code>if</code> on line 25 +<code>nr</code> and <code>er</code> to hold the return values from the call to <code>f.Read</code>. (The <code>if</code> on line 25 has the same idea.) The <code>switch</code> statement is general: it evaluates the cases from top to bottom looking for the first case that matches the value; the case expressions don't need to be constants or even integers, as long as @@ -778,14 +778,14 @@ in a <code>for</code> statement, a missing value means <code>true</code>. In fa is a form of <code>if-else</code> chain. While we're here, it should be mentioned that in <code>switch</code> statements each <code>case</code> has an implicit <code>break</code>. <p> -Line 25 calls <code>Write()</code> by slicing the incoming buffer, which is itself a slice. +Line 25 calls <code>Write</code> by slicing the incoming buffer, which is itself a slice. Slices provide the standard Go way to handle I/O buffers. <p> Now let's make a variant of <code>cat</code> that optionally does <code>rot13</code> on its input. It's easy to do by just processing the bytes, but instead we will exploit Go's notion of an <i>interface</i>. <p> -The <code>cat()</code> subroutine uses only two methods of <code>f</code>: <code>Read()</code> and <code>String()</code>, +The <code>cat</code> subroutine uses only two methods of <code>f</code>: <code>Read</code> and <code>String</code>, so let's start by defining an interface that has exactly those two methods. Here is code from <code>progs/cat_rot13.go</code>: <p> @@ -810,11 +810,11 @@ we have a second implementation of the <code>reader</code> interface. 31 type rotate13 struct { 32 source reader 33 } -<p> + 35 func newRotate13(source reader) *rotate13 { 36 return &rotate13{source} 37 } -<p> + 39 func (r13 *rotate13) Read(b []byte) (ret int, err os.Error) { 40 r, e := r13.source.Read(b) 41 for i := 0; i < r; i++ { @@ -822,7 +822,7 @@ we have a second implementation of the <code>reader</code> interface. 43 } 44 return r, e 45 } -<p> + 47 func (r13 *rotate13) String() string { 48 return r13.source.String() 49 } @@ -837,13 +837,13 @@ To use the new feature, we define a flag: 14 var rot13Flag = flag.Bool("rot13", false, "rot13 the input") </pre> <p> -and use it from within a mostly unchanged <code>cat()</code> function: +and use it from within a mostly unchanged <code>cat</code> function: <p> <pre> <!-- progs/cat_rot13.go /func.cat/ /^}/ --> 52 func cat(r reader) { 53 const NBUF = 512 54 var buf [NBUF]byte -<p> + 56 if *rot13Flag { 57 r = newRotate13(r) 58 } @@ -865,7 +865,7 @@ and use it from within a mostly unchanged <code>cat()</code> function: 74 } </pre> <p> -(We could also do the wrapping in <code>main</code> and leave <code>cat()</code> mostly alone, except +(We could also do the wrapping in <code>main</code> and leave <code>cat</code> mostly alone, except for changing the type of the argument; consider that an exercise.) Lines 56 through 58 set it all up: If the <code>rot13</code> flag is true, wrap the <code>reader</code> we received into a <code>rotate13</code> and proceed. Note that the interface variables @@ -936,7 +936,7 @@ arrays of integers, strings, etc.; here's the code for arrays of <code>int</code <p> <pre> <!-- progs/sort.go /type.*IntArray/ /Swap/ --> 33 type IntArray []int -<p> + 35 func (p IntArray) Len() int { return len(p) } 36 func (p IntArray) Less(i, j int) bool { return p[i] < p[j] } 37 func (p IntArray) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -969,11 +969,11 @@ to implement the three methods for that type, like this: 32 shortName string 33 longName string 34 } -<p> + 36 type dayArray struct { 37 data []*day 38 } -<p> + 40 func (p *dayArray) Len() int { return len(p.data) } 41 func (p *dayArray) Less(i, j int) bool { return p.data[i].num < p.data[j].num } 42 func (p *dayArray) Swap(i, j int) { p.data[i], p.data[j] = p.data[j], p.data[i] } @@ -1054,7 +1054,7 @@ to that of the <code>Printf</code> call above. </pre> <p> If you have your own type you'd like <code>Printf</code> or <code>Print</code> to format, -just give it a <code>String()</code> method that returns a string. The print +just give it a <code>String</code> method that returns a string. The print routines will examine the value to inquire whether it implements the method and if so, use it rather than some other formatting. Here's a simple example. @@ -1064,25 +1064,25 @@ Here's a simple example. 10 a int 11 b string 12 } -<p> + 14 func (t *testType) String() string { 15 return fmt.Sprint(t.a) + " " + t.b 16 } -<p> + 18 func main() { 19 t := &testType{77, "Sunset Strip"} 20 fmt.Println(t) 21 } </pre> <p> -Since <code>*testType</code> has a <code>String()</code> method, the +Since <code>*testType</code> has a <code>String</code> method, the default formatter for that type will use it and produce the output <p> <pre> 77 Sunset Strip </pre> <p> -Observe that the <code>String()</code> method calls <code>Sprint</code> (the obvious Go +Observe that the <code>String</code> method calls <code>Sprint</code> (the obvious Go variant that returns a string) to do its formatting; special formatters can use the <code>fmt</code> library recursively. <p> @@ -1095,7 +1095,7 @@ and such, but that's getting a little off the main thread so we'll leave it as an exploration exercise. <p> You might ask, though, how <code>Printf</code> can tell whether a type implements -the <code>String()</code> method. Actually what it does is ask if the value can +the <code>String</code> method. Actually what it does is ask if the value can be converted to an interface variable that implements the method. Schematically, given a value <code>v</code>, it does this: <p> @@ -1140,7 +1140,7 @@ interface type defined in the <code>io</code> library: <p> (This interface is another conventional name, this time for <code>Write</code>; there are also <code>io.Reader</code>, <code>io.ReadWriter</code>, and so on.) -Thus you can call <code>Fprintf</code> on any type that implements a standard <code>Write()</code> +Thus you can call <code>Fprintf</code> on any type that implements a standard <code>Write</code> method, not just files but also network channels, buffers, whatever you want. <p> @@ -1346,7 +1346,7 @@ code that invokes the operation and responds to the request: <p> <pre> <!-- progs/server.go /type.binOp/ /^}/ --> 14 type binOp func(a, b int) int -<p> + 16 func run(op binOp, req *request) { 17 reply := op(req.a, req.b) 18 req.replyc <- reply diff --git a/doc/go_tutorial.txt b/doc/go_tutorial.txt index 3e0b21d99..513190ef2 100644 --- a/doc/go_tutorial.txt +++ b/doc/go_tutorial.txt @@ -271,7 +271,7 @@ Using slices one can write this function (from "sum.go"): --PROG progs/sum.go /sum/ /^}/ -Note how the return type ("int") is defined for "sum()" by stating it +Note how the return type ("int") is defined for "sum" by stating it after the parameter list. To call the function, we slice the array. This intricate call (we'll show @@ -295,7 +295,7 @@ There are also maps, which you can initialize like this: m := map[string]int{"one":1 , "two":2} -The built-in function "len()", which returns number of elements, +The built-in function "len", which returns number of elements, makes its first appearance in "sum". It works on strings, arrays, slices, maps, and channels. @@ -320,7 +320,7 @@ An Interlude about Allocation Most types in Go are values. If you have an "int" or a "struct" or an array, assignment copies the contents of the object. -To allocate a new variable, use "new()", which +To allocate a new variable, use the built-in function "new", which returns a pointer to the allocated storage. type T struct { a, b int } @@ -333,7 +333,7 @@ or the more idiomatic Some types—maps, slices, and channels (see below)—have reference semantics. If you're holding a slice or a map and you modify its contents, other variables referencing the same underlying data will see the modification. For these three -types you want to use the built-in function "make()": +types you want to use the built-in function "make": m := make(map[string]int) @@ -343,11 +343,11 @@ If you just declare the map, as in var m map[string]int it creates a "nil" reference that cannot hold anything. To use the map, -you must first initialize the reference using "make()" or by assignment from an +you must first initialize the reference using "make" or by assignment from an existing map. Note that "new(T)" returns type "*T" while "make(T)" returns type -"T". If you (mistakenly) allocate a reference object with "new()", +"T". If you (mistakenly) allocate a reference object with "new" rather than "make", you receive a pointer to a nil reference, equivalent to declaring an uninitialized variable and taking its address. @@ -525,7 +525,7 @@ Building on the "file" package, here's a simple version of the Unix utility "cat By now this should be easy to follow, but the "switch" statement introduces some new features. Like a "for" loop, an "if" or "switch" can include an initialization statement. The "switch" on line 18 uses one to create variables -"nr" and "er" to hold the return values from "f.Read()". (The "if" on line 25 +"nr" and "er" to hold the return values from the call to "f.Read". (The "if" on line 25 has the same idea.) The "switch" statement is general: it evaluates the cases from top to bottom looking for the first case that matches the value; the case expressions don't need to be constants or even integers, as long as @@ -537,14 +537,14 @@ in a "for" statement, a missing value means "true". In fact, such a "switch" is a form of "if-else" chain. While we're here, it should be mentioned that in "switch" statements each "case" has an implicit "break". -Line 25 calls "Write()" by slicing the incoming buffer, which is itself a slice. +Line 25 calls "Write" by slicing the incoming buffer, which is itself a slice. Slices provide the standard Go way to handle I/O buffers. Now let's make a variant of "cat" that optionally does "rot13" on its input. It's easy to do by just processing the bytes, but instead we will exploit Go's notion of an <i>interface</i>. -The "cat()" subroutine uses only two methods of "f": "Read()" and "String()", +The "cat" subroutine uses only two methods of "f": "Read" and "String", so let's start by defining an interface that has exactly those two methods. Here is code from "progs/cat_rot13.go": @@ -568,11 +568,11 @@ To use the new feature, we define a flag: --PROG progs/cat_rot13.go /rot13Flag/ -and use it from within a mostly unchanged "cat()" function: +and use it from within a mostly unchanged "cat" function: --PROG progs/cat_rot13.go /func.cat/ /^}/ -(We could also do the wrapping in "main" and leave "cat()" mostly alone, except +(We could also do the wrapping in "main" and leave "cat" mostly alone, except for changing the type of the argument; consider that an exercise.) Lines 56 through 58 set it all up: If the "rot13" flag is true, wrap the "reader" we received into a "rotate13" and proceed. Note that the interface variables @@ -700,19 +700,19 @@ to that of the "Printf" call above. --PROG progs/print.go 'NR==21' 'NR==22' If you have your own type you'd like "Printf" or "Print" to format, -just give it a "String()" method that returns a string. The print +just give it a "String" method that returns a string. The print routines will examine the value to inquire whether it implements the method and if so, use it rather than some other formatting. Here's a simple example. --PROG progs/print_string.go 'NR==9' END -Since "*testType" has a "String()" method, the +Since "*testType" has a "String" method, the default formatter for that type will use it and produce the output 77 Sunset Strip -Observe that the "String()" method calls "Sprint" (the obvious Go +Observe that the "String" method calls "Sprint" (the obvious Go variant that returns a string) to do its formatting; special formatters can use the "fmt" library recursively. @@ -725,7 +725,7 @@ and such, but that's getting a little off the main thread so we'll leave it as an exploration exercise. You might ask, though, how "Printf" can tell whether a type implements -the "String()" method. Actually what it does is ask if the value can +the "String" method. Actually what it does is ask if the value can be converted to an interface variable that implements the method. Schematically, given a value "v", it does this: @@ -764,7 +764,7 @@ interface type defined in the "io" library: (This interface is another conventional name, this time for "Write"; there are also "io.Reader", "io.ReadWriter", and so on.) -Thus you can call "Fprintf" on any type that implements a standard "Write()" +Thus you can call "Fprintf" on any type that implements a standard "Write" method, not just files but also network channels, buffers, whatever you want. diff --git a/doc/godocs.js b/doc/godocs.js index 2b3ab065f..946c4c39f 100644 --- a/doc/godocs.js +++ b/doc/godocs.js @@ -28,6 +28,10 @@ function godocs_onload() { function godocs_bindSearchEvents() { var search = document.getElementById('search'); + if (!search) { + // no search box (index disabled) + return; + } function clearInactive() { if (search.className == "inactive") { search.value = ""; @@ -100,7 +104,7 @@ function godocs_generateTOC() { } } - if (!toc_items.length) { return; } + if (toc_items.length <= 1) { return; } var dl1 = document.createElement('dl'); var dl2 = document.createElement('dl'); diff --git a/doc/htmlgen.go b/doc/htmlgen.go index 4d68767c3..3a8feb8bc 100644 --- a/doc/htmlgen.go +++ b/doc/htmlgen.go @@ -5,7 +5,7 @@ // Process plain text into HTML. // - h2's are made from lines followed by a line "----\n" // - tab-indented blocks become <pre> blocks -// - blank lines become <p> marks +// - blank lines become <p> marks (except inside <pre> tags) // - "quoted strings" become <code>quoted strings</code> package main @@ -35,9 +35,9 @@ var ( func main() { read() headings() - paragraphs() coalesce(preStart, foldPre) coalesce(tab, foldTabs) + paragraphs() quotes() write() } diff --git a/doc/root.html b/doc/root.html index f98f9c243..8d76928c8 100644 --- a/doc/root.html +++ b/doc/root.html @@ -60,7 +60,7 @@ google.setOnLoadCallback(loadFeed); <div id="content-rotating"> <div id="content-videos"> <h2>Go Videos <span class="more">| <a href="/doc/docs.html#videos_talks">More...</a></span></h2> - <a class="video" href="http://osdc.blip.tv/file/4432146/"><img src="/doc/play_overlay.png" class="thumbnail _004" /> <span class="caption title">“Practical Go Programming”</span> <span class="caption description">OSDC Tech Talk</span></a> + <a class="video" href="http://www.youtube.com/watch?v=-i0hat7pdpk"><img src="/doc/play_overlay.png" class="thumbnail _005" /> <span class="caption title">“Writing Web Apps in Go”</span> <span class="caption description">Google I/O 2011</span></a> </div> <h2>Go Blog <span class="more">| <a href="http://blog.golang.org/">More...</a></span></h2> <div id="content-blog"> @@ -78,9 +78,10 @@ google.setOnLoadCallback(loadFeed); <li><a href="/doc/effective_go.html">Effective Go</a></li> <li><a href="/doc/go_faq.html">Go FAQ</a></li> <li><a href="/doc/docs.html">Other Documentation</a></li> + <li><a href="http://code.google.com/appengine/docs/go/">Go for Google App Engine</a> <sup class="new">New!</sup></li> </ul> </div> - <div id="resources-contributors" class="resources"> + <div id="resources-contributors" > <h3>For developers:</h3> <ul> <li><a href="http://godashboard.appspot.com/package">Package Dashboard</a></li> diff --git a/doc/talks/io2011/Real_World_Go.pdf b/doc/talks/io2011/Real_World_Go.pdf Binary files differnew file mode 100644 index 000000000..2a187116b --- /dev/null +++ b/doc/talks/io2011/Real_World_Go.pdf diff --git a/doc/talks/io2011/Writing_Web_Apps_in_Go.pdf b/doc/talks/io2011/Writing_Web_Apps_in_Go.pdf Binary files differnew file mode 100644 index 000000000..ca4702ee9 --- /dev/null +++ b/doc/talks/io2011/Writing_Web_Apps_in_Go.pdf diff --git a/doc/video-005.jpg b/doc/video-005.jpg Binary files differnew file mode 100644 index 000000000..32371581f --- /dev/null +++ b/doc/video-005.jpg diff --git a/lib/codereview/codereview.py b/lib/codereview/codereview.py index 36d7df199..ba2bbbaad 100644 --- a/lib/codereview/codereview.py +++ b/lib/codereview/codereview.py @@ -45,10 +45,23 @@ import stat import subprocess import threading from HTMLParser import HTMLParser + +# The standard 'json' package is new in Python 2.6. +# Before that it was an external package named simplejson. try: - from xml.etree import ElementTree as ET -except: - from elementtree import ElementTree as ET + # Standard location in 2.6 and beyond. + import json +except Exception, e: + try: + # Conventional name for earlier package. + import simplejson as json + except: + try: + # Was also bundled with django, which is commonly installed. + from django.utils import simplejson as json + except: + # We give up. + raise e try: hgversion = util.version() @@ -502,14 +515,16 @@ def LoadCL(ui, repo, name, web=True): else: cl = CL(name) if web: - try: - f = GetSettings(name) - except: - return None, "cannot load CL %s from code review server: %s" % (name, ExceptionDetail()) - if 'reviewers' not in f: + set_status("getting issue metadata from web") + d = JSONGet(ui, "/api/" + name + "?messages=true") + set_status(None) + if d is None: + return None, "cannot load CL %s from server" % (name,) + if 'owner_email' not in d or 'issue' not in d or str(d['issue']) != name: return None, "malformed response loading CL data from code review server" - cl.reviewer = SplitCommaSpace(f['reviewers']) - cl.cc = SplitCommaSpace(f['cc']) + cl.dict = d + cl.reviewer = d.get('reviewers', []) + cl.cc = d.get('cc', []) if cl.local and cl.copied_from and cl.desc: # local copy of CL written by someone else # and we saved a description. use that one, @@ -517,9 +532,10 @@ def LoadCL(ui, repo, name, web=True): # before doing hg submit. pass else: - cl.desc = f['description'] + cl.desc = d.get('description', "") cl.url = server_url_base + name cl.web = True + cl.private = d.get('private', False) != False set_status("loaded CL " + name) return cl, '' @@ -1320,8 +1336,6 @@ def clpatch_or_undo(ui, repo, clname, opts, mode): return err if patch == emptydiff: return "codereview issue %s has no diff" % clname - if not repo[vers]: - return "codereview issue %s is newer than the current repository; hg sync" % clname # find current hg version (hg identify) ctx = repo[None] @@ -1330,7 +1344,14 @@ def clpatch_or_undo(ui, repo, clname, opts, mode): # if version does not match the patch version, # try to update the patch line numbers. - if id != vers: + if vers != "" and id != vers: + # "vers in repo" gives the wrong answer + # on some versions of Mercurial. Instead, do the actual + # lookup and catch the exception. + try: + repo[vers].description() + except: + return "local repository is out of date; sync to get %s" % (vers) patch, err = portPatch(repo, patch, vers, id) if err != "": return "codereview issue %s is out of date: %s (%s->%s)" % (clname, err, vers, id) @@ -2000,100 +2021,94 @@ class FormParser(HTMLParser): if self.curdata is not None: self.curdata += data -# XML parser -def XMLGet(ui, path): +def JSONGet(ui, path): try: - data = MySend(path, force_auth=False); + data = MySend(path, force_auth=False) + typecheck(data, str) + d = fix_json(json.loads(data)) except: - ui.warn("XMLGet %s: %s\n" % (path, ExceptionDetail())) + ui.warn("JSONGet %s: %s\n" % (path, ExceptionDetail())) return None - return ET.XML(data) + return d + +# Clean up json parser output to match our expectations: +# * all strings are UTF-8-encoded str, not unicode. +# * missing fields are missing, not None, +# so that d.get("foo", defaultvalue) works. +def fix_json(x): + if type(x) in [str, int, float, bool, type(None)]: + pass + elif type(x) is unicode: + x = x.encode("utf-8") + elif type(x) is list: + for i in range(len(x)): + x[i] = fix_json(x[i]) + elif type(x) is dict: + todel = [] + for k in x: + if x[k] is None: + todel.append(k) + else: + x[k] = fix_json(x[k]) + for k in todel: + del x[k] + else: + raise util.Abort("unknown type " + str(type(x)) + " in fix_json") + if type(x) is str: + x = x.replace('\r\n', '\n') + return x def IsRietveldSubmitted(ui, clname, hex): - feed = XMLGet(ui, "/rss/issue/" + clname) - if feed is None: + dict = JSONGet(ui, "/api/" + clname + "?messages=true") + if dict is None: return False - for sum in feed.findall("{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}summary"): - text = sum.text.strip() + for msg in dict.get("messages", []): + text = msg.get("text", "") m = re.match('\*\*\* Submitted as [^*]*?([0-9a-f]+) \*\*\*', text) if m is not None and len(m.group(1)) >= 8 and hex.startswith(m.group(1)): return True return False -def IsRietveldMailed(ui, clname): - feed = XMLGet(ui, "/rss/issue/" + clname) - if feed is None: - return False - for sum in feed.findall("{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}summary"): - text = sum.text.strip() - if re.match("I'd like you to review this change", text): +def IsRietveldMailed(cl): + for msg in cl.dict.get("messages", []): + if msg.get("text", "").find("I'd like you to review this change") >= 0: return True return False def DownloadCL(ui, repo, clname): set_status("downloading CL " + clname) - cl, err = LoadCL(ui, repo, clname) + cl, err = LoadCL(ui, repo, clname, web=True) if err != "": return None, None, None, "error loading CL %s: %s" % (clname, err) - # Grab RSS feed to learn about CL - feed = XMLGet(ui, "/rss/issue/" + clname) - if feed is None: - return None, None, None, "cannot download CL" - # Find most recent diff - diff = None - prefix = 'http://' + server + '/' + diffs = cl.dict.get("patchsets", []) + if not diffs: + return None, None, None, "CL has no patch sets" + patchid = diffs[-1] + + patchset = JSONGet(ui, "/api/" + clname + "/" + str(patchid)) + if patchset is None: + return None, None, None, "error loading CL patchset %s/%d" % (clname, patchid) + if patchset.get("patchset", 0) != patchid: + return None, None, None, "malformed patchset information" + vers = "" - for entry in feed.findall("{http://www.w3.org/2005/Atom}entry"): - thisVers = "" - for title in entry.findall("{http://www.w3.org/2005/Atom}title"): - m = re.search('diff -r ([0-9a-f]+) ', title.text) - if m: - thisVers = m.group(1) - if thisVers == "": - continue - for link in entry.findall("{http://www.w3.org/2005/Atom}link"): - if link.get('rel') != 'alternate': - continue - text = link.get('href') - if not text.startswith(prefix) or not text.endswith('.diff'): - continue - diff = text[len(prefix)-1:] - vers = thisVers - if diff is None: - return None, None, None, "CL has no diff" - diffdata = MySend(diff, force_auth=False) - - # Find author - first entry will be author who created CL. - nick = None - for author in feed.findall("{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}author/{http://www.w3.org/2005/Atom}name"): - nick = author.text.strip() - break - if not nick: - return None, None, None, "CL has no author" - - # The author is just a nickname: get the real email address. - try: - # want URL-encoded nick, but without a=, and rietveld rejects + for %20. - url = "/user_popup/" + urllib.urlencode({"a": nick})[2:].replace("+", "%20") - data = MySend(url, force_auth=False) - except: - ui.warn("error looking up %s: %s\n" % (nick, ExceptionDetail())) - cl.copied_from = nick+"@needtofix" - return cl, vers, diffdata, "" - match = re.match(r"<b>(.*) \((.*)\)</b>", data) - if not match: - return None, None, "error looking up %s: cannot parse result %s" % (nick, repr(data)) - if match.group(1) != nick and match.group(2) != nick: - return None, None, "error looking up %s: got info for %s, %s" % (nick, match.group(1), match.group(2)) - email = match.group(1) + msg = patchset.get("message", "").split() + if len(msg) >= 3 and msg[0] == "diff" and msg[1] == "-r": + vers = msg[2] + diff = "/download/issue" + clname + "_" + str(patchid) + ".diff" + diffdata = MySend(diff, force_auth=False) + # Print warning if email is not in CONTRIBUTORS file. + email = cl.dict.get("owner_email", "") + if not email: + return None, None, None, "cannot find owner for %s" % (clname) him = FindContributor(ui, repo, email) me = FindContributor(ui, repo, None) if him == me: - cl.mailed = IsRietveldMailed(ui, clname) + cl.mailed = IsRietveldMailed(cl) else: cl.copied_from = email @@ -2192,25 +2207,6 @@ def GetForm(url): m[k.encode("utf-8")] = v.replace("\r\n", "\n").encode("utf-8") return m -# Fetch the settings for the CL, like reviewer and CC list, by -# scraping the Rietveld editing forms. -def GetSettings(issue): - set_status("getting issue metadata from web") - # The /issue/edit page has everything but only the - # CL owner is allowed to fetch it (and submit it). - f = None - try: - f = GetForm("/" + issue + "/edit") - except: - pass - if not f or 'reviewers' not in f: - # Maybe we're not the CL owner. Fall back to the - # /publish page, which has the reviewer and CC lists, - # and then fetch the description separately. - f = GetForm("/" + issue + "/publish") - f['description'] = MySend("/"+issue+"/description", force_auth=False) - return f - def EditDesc(issue, subject=None, desc=None, reviewers=None, cc=None, closed=False, private=False): set_status("uploading change to description") form_fields = GetForm("/" + issue + "/edit") diff --git a/lib/godoc/godoc.html b/lib/godoc/godoc.html index 837f53c48..bd53f2615 100644 --- a/lib/godoc/godoc.html +++ b/lib/godoc/godoc.html @@ -33,7 +33,9 @@ <a href="/pkg/">Packages</a> <span class="sep">|</span> <a href="/cmd/">Commands</a> <span class="sep">|</span> <a href="/doc/go_spec.html">Specification</a> + {.section SearchBox} <input id="search" type="search" name="q" value="{.section Query}{Query|html-esc}{.end}" class="{.section Query}{.or}inactive{.end}" placeholder="code search" results="0" /> + {.end} </form> </div> </div> @@ -54,11 +56,9 @@ <span class="subtitle">{@|html-esc}</span> {.end} - <p> <!-- The Table of Contents is automatically inserted in this <div>. Do not delete this <div>. --> <div id="nav"></div> - </p> <!-- Content is HTML-escaped elsewhere --> {Content} diff --git a/lib/godoc/package.html b/lib/godoc/package.html index 4205f9315..cb3ffa2a7 100644 --- a/lib/godoc/package.html +++ b/lib/godoc/package.html @@ -91,6 +91,11 @@ </p> {.end} {.section Dirs} + <p class="detail"> + Need more packages? The + <a href="http://godashboard.appspot.com/package">Package Dashboard</a> + provides a list of <a href="/cmd/goinstall/">goinstallable</a> packages. + </p> {# DirList entries are numbers and strings - no need for FSet} <h2 id="Subdirectories">Subdirectories</h2> <p> diff --git a/misc/IntelliJIDEA/Go.xml b/misc/IntelliJIDEA/Go.xml new file mode 100644 index 000000000..09265a2e0 --- /dev/null +++ b/misc/IntelliJIDEA/Go.xml @@ -0,0 +1,98 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Copyright 2011 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. + +Copy this custom language definition & configuration file to + * Mac : ~/Library/Preferences/IntelliJIdea10/filetypes/ + * Linux & Windows : ~/.IntelliJIdea10/config/filetypes/ +--> + +<filetype binary="false" default_extension="" description="Go" name="Go"> + <highlighting> + <options> + <option name="LINE_COMMENT" value="//"/> + <option name="COMMENT_START" value="/*"/> + <option name="COMMENT_END" value="*/"/> + <option name="HEX_PREFIX" value="0x"/> + <option name="NUM_POSTFIXES" value=""/> + <option name="HAS_BRACKETS" value="true"/> + <option name="HAS_BRACES" value="true"/> + <option name="HAS_PARENS" value="true"/> + <option name="HAS_STRING_ESCAPES" value="true"/> + </options> + <keywords ignore_case="false"> + <keyword name="break"/> + <keyword name="case"/> + <keyword name="chan"/> + <keyword name="const"/> + <keyword name="continue"/> + <keyword name="default"/> + <keyword name="defer"/> + <keyword name="else"/> + <keyword name="fallthrough"/> + <keyword name="for"/> + <keyword name="func"/> + <keyword name="go"/> + <keyword name="goto"/> + <keyword name="if"/> + <keyword name="import"/> + <keyword name="interface"/> + <keyword name="map"/> + <keyword name="package"/> + <keyword name="range"/> + <keyword name="return"/> + <keyword name="select"/> + <keyword name="struct"/> + <keyword name="switch"/> + <keyword name="type"/> + <keyword name="var"/> + </keywords> + <keywords2> + <keyword name="bool"/> + <keyword name="byte"/> + <keyword name="complex64"/> + <keyword name="complex128"/> + <keyword name="float32"/> + <keyword name="float64"/> + <keyword name="int"/> + <keyword name="int8"/> + <keyword name="int16"/> + <keyword name="int32"/> + <keyword name="int64"/> + <keyword name="string"/> + <keyword name="uint"/> + <keyword name="uint8"/> + <keyword name="uint16"/> + <keyword name="uint32"/> + <keyword name="uint64"/> + <keyword name="uintptr"/> + </keywords2> + <keywords3> + <keyword name="append"/> + <keyword name="cap"/> + <keyword name="close"/> + <keyword name="complex"/> + <keyword name="copy"/> + <keyword name="imag"/> + <keyword name="len"/> + <keyword name="make"/> + <keyword name="new"/> + <keyword name="panic"/> + <keyword name="print"/> + <keyword name="println"/> + <keyword name="real"/> + <keyword name="recover"/> + </keywords3> + <keywords4> + <keyword name="false"/> + <keyword name="iota"/> + <keyword name="nil"/> + <keyword name="true"/> + </keywords4> + </highlighting> + <extensionMap> + <mapping ext="go"/> + </extensionMap> +</filetype> diff --git a/misc/cgo/gmp/gmp.go b/misc/cgo/gmp/gmp.go index f7bbe9c51..7faa71b69 100644 --- a/misc/cgo/gmp/gmp.go +++ b/misc/cgo/gmp/gmp.go @@ -86,9 +86,8 @@ explicitly in Go to pointers to arrays, as they do (implicitly) in C. Garbage collection is the big problem. It is fine for the Go world to have pointers into the C world and to free those pointers when they -are no longer needed. To help, the garbage collector calls an -object's destroy() method prior to collecting it. C pointers can be -wrapped by Go objects with appropriate destroy methods. +are no longer needed. To help, the Go code can define Go objects +holding the C pointers and use runtime.SetFinalizer on those Go objects. It is much more difficult for the C world to have pointers into the Go world, because the Go garbage collector is unaware of the memory diff --git a/misc/cgo/test/Makefile b/misc/cgo/test/Makefile index 893540d97..43c45f416 100644 --- a/misc/cgo/test/Makefile +++ b/misc/cgo/test/Makefile @@ -10,6 +10,7 @@ CGOFILES=\ align.go\ basic.go\ callback.go\ + env.go\ issue1222.go\ issue1328.go\ issue1560.go\ diff --git a/misc/cgo/test/align.go b/misc/cgo/test/align.go index 2d2979595..07ab9ef50 100644 --- a/misc/cgo/test/align.go +++ b/misc/cgo/test/align.go @@ -58,7 +58,7 @@ import ( "testing" ) -func TestAlign(t *testing.T) { +func testAlign(t *testing.T) { var evt C.SDL_KeyboardEvent C.makeEvent(&evt) if C.same(&evt, evt.typ, evt.which, evt.state, evt.keysym.scancode, evt.keysym.sym, evt.keysym.mod, evt.keysym.unicode) == 0 { diff --git a/misc/cgo/test/basic.go b/misc/cgo/test/basic.go index a94074c52..b9d0953bd 100644 --- a/misc/cgo/test/basic.go +++ b/misc/cgo/test/basic.go @@ -90,31 +90,31 @@ func Atol(s string) int { return int(n) } -func TestConst(t *testing.T) { +func testConst(t *testing.T) { C.myConstFunc(nil, 0, nil) } -func TestEnum(t *testing.T) { +func testEnum(t *testing.T) { if C.Enum1 != 1 || C.Enum2 != 2 { t.Error("bad enum", C.Enum1, C.Enum2) } } -func TestAtol(t *testing.T) { +func testAtol(t *testing.T) { l := Atol("123") if l != 123 { t.Error("Atol 123: ", l) } } -func TestErrno(t *testing.T) { +func testErrno(t *testing.T) { n, err := Strtol("asdf", 123) if n != 0 || err != os.EINVAL { t.Error("Strtol: ", n, err) } } -func TestMultipleAssign(t *testing.T) { +func testMultipleAssign(t *testing.T) { p := C.CString("234") n, m := C.strtol(p, nil, 345), C.strtol(p, nil, 10) if n != 0 || m != 234 { diff --git a/misc/cgo/test/callback.go b/misc/cgo/test/callback.go index 450a7cbf2..3edee9758 100644 --- a/misc/cgo/test/callback.go +++ b/misc/cgo/test/callback.go @@ -27,7 +27,7 @@ func goCallback(p unsafe.Pointer) { (*(*func())(unsafe.Pointer(&p)))() } -func TestCallback(t *testing.T) { +func testCallback(t *testing.T) { var x = false nestedCall(func() { x = true }) if !x { @@ -35,13 +35,13 @@ func TestCallback(t *testing.T) { } } -func TestCallbackGC(t *testing.T) { +func testCallbackGC(t *testing.T) { nestedCall(runtime.GC) } func lockedOSThread() bool // in runtime.c -func TestCallbackPanic(t *testing.T) { +func testCallbackPanic(t *testing.T) { // Make sure panic during callback unwinds properly. if lockedOSThread() { t.Fatal("locked OS thread on entry to TestCallbackPanic") @@ -62,14 +62,14 @@ func TestCallbackPanic(t *testing.T) { panic("nestedCall returned") } -func TestCallbackPanicLoop(t *testing.T) { +func testCallbackPanicLoop(t *testing.T) { // Make sure we don't blow out m->g0 stack. for i := 0; i < 100000; i++ { TestCallbackPanic(t) } } -func TestCallbackPanicLocked(t *testing.T) { +func testCallbackPanicLocked(t *testing.T) { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -94,7 +94,7 @@ func TestCallbackPanicLocked(t *testing.T) { // Callback with zero arguments used to make the stack misaligned, // which broke the garbage collector and other things. -func TestZeroArgCallback(t *testing.T) { +func testZeroArgCallback(t *testing.T) { defer func() { s := recover() if s != nil { @@ -118,7 +118,7 @@ func goFoo() { func variadic(x ...interface{}) {} -func TestBlocking(t *testing.T) { +func testBlocking(t *testing.T) { c := make(chan int) go func() { for i := 0; i < 10; i++ { diff --git a/misc/cgo/test/cgo_test.go b/misc/cgo/test/cgo_test.go index 967dc0e92..94fba15db 100644 --- a/misc/cgo/test/cgo_test.go +++ b/misc/cgo/test/cgo_test.go @@ -1,5 +1,28 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package cgotest -// dummy file so gotest thinks there are tests. -// the actual tests are in the main go files, next -// to the code they test. +import "testing" + +// The actual test functions are in non-_test.go files +// so that they can use cgo (import "C"). +// These wrappers are here for gotest to find. + +func TestAlign(t *testing.T) { testAlign(t) } +func TestConst(t *testing.T) { testConst(t) } +func TestEnum(t *testing.T) { testEnum(t) } +func TestAtol(t *testing.T) { testAtol(t) } +func TestErrno(t *testing.T) { testErrno(t) } +func TestMultipleAssign(t *testing.T) { testMultipleAssign(t) } +func TestCallback(t *testing.T) { testCallback(t) } +func TestCallbackGC(t *testing.T) { testCallbackGC(t) } +func TestCallbackPanic(t *testing.T) { testCallbackPanic(t) } +func TestCallbackPanicLoop(t *testing.T) { testCallbackPanicLoop(t) } +func TestCallbackPanicLocked(t *testing.T) { testCallbackPanicLocked(t) } +func TestZeroArgCallback(t *testing.T) { testZeroArgCallback(t) } +func TestBlocking(t *testing.T) { testBlocking(t) } +func Test1328(t *testing.T) { test1328(t) } +func TestParallelSleep(t *testing.T) { testParallelSleep(t) } +func TestSetEnv(t *testing.T) { testSetEnv(t) } diff --git a/misc/cgo/test/env.go b/misc/cgo/test/env.go new file mode 100644 index 000000000..1fb4e684c --- /dev/null +++ b/misc/cgo/test/env.go @@ -0,0 +1,32 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgotest + +/* +#include <stdlib.h> +*/ +import "C" +import ( + "os" + "testing" + "unsafe" +) + +// This is really an os package test but here for convenience. +func testSetEnv(t *testing.T) { + const key = "CGO_OS_TEST_KEY" + const val = "CGO_OS_TEST_VALUE" + os.Setenv(key, val) + keyc := C.CString(key) + defer C.free(unsafe.Pointer(keyc)) + v := C.getenv(keyc) + if v == (*C.char)(unsafe.Pointer(uintptr(0))) { + t.Fatal("getenv returned NULL") + } + vs := C.GoString(v) + if vs != val { + t.Fatalf("getenv() = %q; want %q", vs, val) + } +} diff --git a/misc/cgo/test/issue1328.go b/misc/cgo/test/issue1328.go index f29d7057e..e01207dd9 100644 --- a/misc/cgo/test/issue1328.go +++ b/misc/cgo/test/issue1328.go @@ -25,6 +25,6 @@ func BackIntoGo() { func xvariadic(x ...interface{}) { } -func Test1328(t *testing.T) { +func test1328(t *testing.T) { C.IntoC() } diff --git a/misc/cgo/test/issue1560.go b/misc/cgo/test/issue1560.go index 75d31c035..e534cce47 100644 --- a/misc/cgo/test/issue1560.go +++ b/misc/cgo/test/issue1560.go @@ -35,7 +35,7 @@ func BackgroundSleep(n int) { }() } -func TestParallelSleep(t *testing.T) { +func testParallelSleep(t *testing.T) { dt := -time.Nanoseconds() parallelSleep(1) dt += time.Nanoseconds() diff --git a/misc/chrome/gophertool/README.txt b/misc/chrome/gophertool/README.txt new file mode 100644 index 000000000..a7c0b4b26 --- /dev/null +++ b/misc/chrome/gophertool/README.txt @@ -0,0 +1,8 @@ +To install: + +1) chrome://extensions/ +2) click "[+] Developer Mode" in top right +3) "Load unpacked extension..." +4) pick $GOROOT/misc/chrome/gophertool + +Done. It'll now auto-reload from source. diff --git a/misc/chrome/gophertool/background.html b/misc/chrome/gophertool/background.html new file mode 100644 index 000000000..058c18142 --- /dev/null +++ b/misc/chrome/gophertool/background.html @@ -0,0 +1,24 @@ +<html> +<!-- + Copyright 2011 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. +--> +<head> +<script src="gopher.js"></script> +<script> + +chrome.omnibox.onInputEntered.addListener(function(t) { + var url = urlForInput(t); + if (url) { + chrome.tabs.getSelected(null, function(tab) { + if (!tab) return; + chrome.tabs.update(tab.id, { "url": url, "selected": true }); + }); + } +}); + +</script> +</head> +</html> + diff --git a/misc/chrome/gophertool/gopher.js b/misc/chrome/gophertool/gopher.js new file mode 100644 index 000000000..847c1c70d --- /dev/null +++ b/misc/chrome/gophertool/gopher.js @@ -0,0 +1,34 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +var numericRE = /^\d+$/; +var commitRE = /^(?:\d+:)?([0-9a-f]{6,20})$/; // e.g "8486:ab29d2698a47" or "ab29d2698a47" +var pkgRE = /^[a-z0-9_\/]+$/; + +function urlForInput(t) { + if (!t) { + return null; + } + + if (numericRE.test(t)) { + if (t < 1000000) { + return "http://code.google.com/p/go/issues/detail?id=" + t; + } + return "http://codereview.appspot.com/" + t + "/"; + } + + var match = commitRE.exec(t); + if (match) { + return "http://code.google.com/p/go/source/detail?r=" + match[1]; + } + + if (pkgRE.test(t)) { + // TODO: make this smarter, using a list of packages + substring matches. + // Get the list from godoc itself in JSON format? + // TODO: prefer localhost:6060 to golang.org if localhost:6060 is responding. + return "http://golang.org/pkg/" + t; + } + + return null; +} diff --git a/misc/chrome/gophertool/gopher.png b/misc/chrome/gophertool/gopher.png Binary files differnew file mode 100644 index 000000000..0d1abb741 --- /dev/null +++ b/misc/chrome/gophertool/gopher.png diff --git a/misc/chrome/gophertool/manifest.json b/misc/chrome/gophertool/manifest.json new file mode 100644 index 000000000..3a2540a86 --- /dev/null +++ b/misc/chrome/gophertool/manifest.json @@ -0,0 +1,17 @@ +{ + "name": "Hacking Gopher", + "version": "1.0", + "description": "Go Hacking utility", + "background_page": "background.html", + "browser_action": { + "default_icon": "gopher.png", + "popup": "popup.html" + }, + "omnibox": { "keyword": "golang" }, + "icons": { + "16": "gopher.png" + }, + "permissions": [ + "tabs" + ] +} diff --git a/misc/chrome/gophertool/popup.html b/misc/chrome/gophertool/popup.html new file mode 100644 index 000000000..ebbc71f3a --- /dev/null +++ b/misc/chrome/gophertool/popup.html @@ -0,0 +1,54 @@ +<html> +<!-- + Copyright 2011 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. +--> +<head> +<script src="gopher.js"></script> +<script> + +function focusinput() { + document.getElementById("inputbox").focus(); +} + +function navigate() { + var box = document.getElementById("inputbox"); + box.focus(); + + var t = box.value; + if (t == "") { + return false; + } + + var success = function(url) { + console.log("matched " + t + " to: " + url) + box.value = ""; + openURL(url); + return false; // cancel form submission + }; + + var url = urlForInput(t); + if (url) { + return success(url); + } + + console.log("no match for text: " + t) + return false; +} + +function openURL(url) { + chrome.tabs.create({ "url": url }) +} + +</script> +</head> +<body onload="focusinput()" style='margin: 0.5em; font-family: sans;'> +<small><a href="#" onclick="openURL('http://code.google.com/p/go/issues/list')">issue</a>, +<a href="#" onclick="openURL('http://codereview.appspot.com/')">codereview</a>, +<a href="#" onclick="openURL('http://code.google.com/p/go/source/list')">commit</a>, or +<a href="#" onclick="openURL('http://golang.org/pkg/')">pkg</a> id/name:</small> +<form style='margin: 0' onsubmit="return navigate();"><nobr><input id="inputbox" size=10 /><input type="submit" value="go" /></nobr></form> +<small>Also: <a href="#" onclick="openURL('http://godashboard.appspot.com/')">buildbots</small> +</body> +</html> diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile index cff47942a..f1d9c5497 100644 --- a/misc/dashboard/builder/Makefile +++ b/misc/dashboard/builder/Makefile @@ -7,7 +7,6 @@ include ../../../src/Make.inc TARG=gobuilder GOFILES=\ exec.go\ - hg.go\ http.go\ main.go\ package.go\ diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go index 7bb7ccbe3..30d8fe948 100644 --- a/misc/dashboard/builder/doc.go +++ b/misc/dashboard/builder/doc.go @@ -14,9 +14,6 @@ It periodically pulls updates from the Go Mercurial repository. When a newer revision is found, Go Builder creates a clone of the repository, runs all.bash, and reports build success or failure to the Go Dashboard. -For a successful build, Go Builder will also run benchmarks -(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard. - For a release revision (a change description that matches "release.YYYY-MM-DD"), Go Builder will create a tar.gz archive of the GOROOT and deliver it to the Go Google Code project's downloads section. @@ -34,8 +31,6 @@ Optional flags: The location of the Go Dashboard application to which Go Builder will report its results. - -bench: Run benchmarks - -release: Build and deliver binary release archive -rev=N: Build revision N and exit @@ -45,7 +40,7 @@ Optional flags: -v: Verbose logging -external: External package builder mode (will not report Go build - state to dashboard, issue releases, or run benchmarks) + state to dashboard or issue releases) The key file should be located at $HOME/.gobuildkey or, for a builder-specific key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64). diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go index 3c6fbdced..a042c5699 100644 --- a/misc/dashboard/builder/exec.go +++ b/misc/dashboard/builder/exec.go @@ -18,59 +18,57 @@ func run(envv []string, dir string, argv ...string) os.Error { if *verbose { log.Println("run", argv) } - bin, err := pathLookup(argv[0]) - if err != nil { - return err - } - p, err := exec.Run(bin, argv, envv, dir, - exec.DevNull, exec.DevNull, exec.PassThrough) - if err != nil { - return err - } - return p.Close() + argv = useBash(argv) + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Env = envv + cmd.Stderr = os.Stderr + return cmd.Run() } // runLog runs a process and returns the combined stdout/stderr, -// as well as writing it to logfile (if specified). -func runLog(envv []string, logfile, dir string, argv ...string) (output string, exitStatus int, err os.Error) { +// as well as writing it to logfile (if specified). It returns +// process combined stdout and stderr output, exit status and error. +// The error returned is nil, if process is started successfully, +// even if exit status is not 0. +func runLog(envv []string, logfile, dir string, argv ...string) (string, int, os.Error) { if *verbose { log.Println("runLog", argv) } - bin, err := pathLookup(argv[0]) - if err != nil { - return - } - p, err := exec.Run(bin, argv, envv, dir, - exec.DevNull, exec.Pipe, exec.MergeWithStdout) - if err != nil { - return - } - defer p.Close() + argv = useBash(argv) + b := new(bytes.Buffer) var w io.Writer = b if logfile != "" { f, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) if err != nil { - return + return "", 0, err } defer f.Close() w = io.MultiWriter(f, b) } - _, err = io.Copy(w, p.Stdout) - if err != nil { - return - } - wait, err := p.Wait(0) + + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Env = envv + cmd.Stdout = w + cmd.Stderr = w + + err := cmd.Run() if err != nil { - return + if ws, ok := err.(*os.Waitmsg); ok { + return b.String(), ws.ExitStatus(), nil + } } - return b.String(), wait.WaitStatus.ExitStatus(), nil + return b.String(), 0, nil } -// Find bin in PATH if a relative or absolute path hasn't been specified -func pathLookup(s string) (string, os.Error) { - if strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../") { - return s, nil +// useBash prefixes a list of args with 'bash' if the first argument +// is a bash script. +func useBash(argv []string) []string { + // TODO(brainman): choose a more reliable heuristic here. + if strings.HasSuffix(argv[0], ".bash") { + argv = append([]string{"bash"}, argv...) } - return exec.LookPath(s) + return argv } diff --git a/misc/dashboard/builder/hg.go b/misc/dashboard/builder/hg.go deleted file mode 100644 index d4310845d..000000000 --- a/misc/dashboard/builder/hg.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "os" - "regexp" - "strconv" - "strings" -) - -type Commit struct { - num int // mercurial revision number - node string // mercurial hash - parent string // hash of commit's parent - user string // author's Name <email> - date string // date of commit - desc string // description -} - -// getCommit returns details about the Commit specified by the revision hash -func getCommit(rev string) (c Commit, err os.Error) { - defer func() { - if err != nil { - err = fmt.Errorf("getCommit: %s: %s", rev, err) - } - }() - parts, err := getCommitParts(rev) - if err != nil { - return - } - num, err := strconv.Atoi(parts[0]) - if err != nil { - return - } - parent := "" - if num > 0 { - prev := strconv.Itoa(num - 1) - if pparts, err := getCommitParts(prev); err == nil { - parent = pparts[1] - } - } - user := strings.Replace(parts[2], "<", "<", -1) - user = strings.Replace(user, ">", ">", -1) - return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil -} - -func getCommitParts(rev string) (parts []string, err os.Error) { - const format = "{rev}>{node}>{author|escape}>{date}>{desc}" - s, _, err := runLog(nil, "", goroot, - "hg", "log", - "--encoding", "utf-8", - "--rev", rev, - "--limit", "1", - "--template", format, - ) - if err != nil { - return - } - return strings.Split(s, ">", 5), nil -} - -var revisionRe = regexp.MustCompile(`([0-9]+):[0-9a-f]+$`) - -// getTag fetches a Commit by finding the first hg tag that matches re. -func getTag(re *regexp.Regexp) (c Commit, tag string, err os.Error) { - o, _, err := runLog(nil, "", goroot, "hg", "tags") - for _, l := range strings.Split(o, "\n", -1) { - tag = re.FindString(l) - if tag == "" { - continue - } - s := revisionRe.FindStringSubmatch(l) - if s == nil { - err = os.NewError("couldn't find revision number") - return - } - c, err = getCommit(s[1]) - return - } - err = os.NewError("no matching tag found") - return -} diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go index dba19ba8f..5e1da0c87 100644 --- a/misc/dashboard/builder/http.go +++ b/misc/dashboard/builder/http.go @@ -6,84 +6,104 @@ package main import ( "bytes" - "encoding/base64" - "encoding/binary" "fmt" "http" "json" "log" "os" - "regexp" "strconv" ) -// getHighWater returns the current highwater revision hash for this builder -func (b *Builder) getHighWater() (rev string, err os.Error) { - url := fmt.Sprintf("http://%s/hw-get?builder=%s", *dashboard, b.name) - r, _, err := http.Get(url) +type param map[string]string + +// dash runs the given method and command on the dashboard. +// If args is not nil, it is the query or post parameters. +// If resp is not nil, dash unmarshals the body as JSON into resp. +func dash(meth, cmd string, resp interface{}, args param) os.Error { + var r *http.Response + var err os.Error + if *verbose { + log.Println("dash", cmd, args) + } + cmd = "http://" + *dashboard + "/" + cmd + vals := make(http.Values) + for k, v := range args { + vals.Add(k, v) + } + switch meth { + case "GET": + if q := vals.Encode(); q != "" { + cmd += "?" + q + } + r, err = http.Get(cmd) + case "POST": + r, err = http.PostForm(cmd, vals) + default: + return fmt.Errorf("unknown method %q", meth) + } if err != nil { - return + return err + } + defer r.Body.Close() + var buf bytes.Buffer + buf.ReadFrom(r.Body) + if resp != nil { + if err = json.Unmarshal(buf.Bytes(), resp); err != nil { + log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err) + return err + } } - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(r.Body) + return nil +} + +func dashStatus(meth, cmd string, args param) os.Error { + var resp struct { + Status string + Error string + } + err := dash(meth, cmd, &resp, args) if err != nil { + return err + } + if resp.Status != "OK" { + return os.NewError("/build: " + resp.Error) + } + return nil +} + +// todo returns the next hash to build. +func (b *Builder) todo() (rev string, err os.Error) { + var resp []struct { + Hash string + } + if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil { return } - r.Body.Close() - return buf.String(), nil + if len(resp) > 0 { + rev = resp[0].Hash + } + return } // recordResult sends build results to the dashboard -func (b *Builder) recordResult(buildLog string, c Commit) os.Error { - return httpCommand("build", map[string]string{ +func (b *Builder) recordResult(buildLog string, hash string) os.Error { + return dash("POST", "build", nil, param{ "builder": b.name, "key": b.key, - "node": c.node, - "parent": c.parent, - "user": c.user, - "date": c.date, - "desc": c.desc, + "node": hash, "log": buildLog, }) } -// match lines like: "package.BechmarkFunc 100000 999 ns/op" -var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op") - -// recordBenchmarks sends benchmark results to the dashboard -func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error { - results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1) - var buf bytes.Buffer - b64 := base64.NewEncoder(base64.StdEncoding, &buf) - for _, r := range results { - for _, s := range r[1:] { - binary.Write(b64, binary.BigEndian, uint16(len(s))) - b64.Write([]byte(s)) - } - } - b64.Close() - return httpCommand("benchmarks", map[string]string{ - "builder": b.name, - "key": b.key, - "node": c.node, - "benchmarkdata": buf.String(), - }) -} - -// getPackages fetches a list of package paths from the dashboard -func getPackages() (pkgs []string, err os.Error) { - r, _, err := http.Get(fmt.Sprintf("http://%v/package?fmt=json", *dashboard)) - if err != nil { - return - } - defer r.Body.Close() - d := json.NewDecoder(r.Body) +// packages fetches a list of package paths from the dashboard +func packages() (pkgs []string, err os.Error) { var resp struct { Packages []struct { Path string } } - if err = d.Decode(&resp); err != nil { + err = dash("GET", "package", &resp, param{"fmt": "json"}) + if err != nil { return } for _, p := range resp.Packages { @@ -93,24 +113,36 @@ func getPackages() (pkgs []string, err os.Error) { } // updatePackage sends package build results and info to the dashboard -func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, c Commit) os.Error { - args := map[string]string{ +func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, hash string) os.Error { + return dash("POST", "package", nil, param{ "builder": b.name, "key": b.key, "path": pkg, "state": strconv.Btoa(state), "log": buildLog, "info": info, - "go_rev": strconv.Itoa(c.num), - } - return httpCommand("package", args) + "go_rev": hash[:12], + }) } -func httpCommand(cmd string, args map[string]string) os.Error { - if *verbose { - log.Println("httpCommand", cmd, args) +// postCommit informs the dashboard of a new commit +func postCommit(key string, l *HgLog) os.Error { + return dashStatus("POST", "commit", param{ + "key": key, + "node": l.Hash, + "date": l.Date, + "user": l.Author, + "parent": l.Parent, + "desc": l.Desc, + }) +} + +// dashboardCommit returns true if the dashboard knows about hash. +func dashboardCommit(hash string) bool { + err := dashStatus("GET", "commit", param{"node": hash}) + if err != nil { + log.Printf("check %s: %s", hash, err) + return false } - url := fmt.Sprintf("http://%v/%v", *dashboard, cmd) - _, err := http.PostForm(url, args) - return err + return true } diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go index d11cbb133..9377fbe32 100644 --- a/misc/dashboard/builder/main.go +++ b/misc/dashboard/builder/main.go @@ -5,7 +5,6 @@ package main import ( - "container/vector" "flag" "fmt" "io/ioutil" @@ -13,16 +12,18 @@ import ( "os" "path" "regexp" + "runtime" "strconv" "strings" "time" + "xml" ) const ( codeProject = "go" codePyScript = "misc/dashboard/googlecode_upload.py" hgUrl = "https://go.googlecode.com/hg/" - waitInterval = 10e9 // time to wait before checking for new revs + waitInterval = 30e9 // time to wait before checking for new revs mkdirPerm = 0750 pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours ) @@ -46,16 +47,10 @@ type Builder struct { codePassword string } -type BenchRequest struct { - builder *Builder - commit Commit - path string -} - var ( buildroot = flag.String("buildroot", path.Join(os.TempDir(), "gobuilder"), "Directory under which to build") + commitFlag = flag.Bool("commit", false, "upload information about new commits") dashboard = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host") - runBenchmarks = flag.Bool("bench", false, "Run benchmarks") buildRelease = flag.Bool("release", false, "Build and upload binary release archives") buildRevision = flag.String("rev", "", "Build specified revision and exit") buildCmd = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)") @@ -67,7 +62,6 @@ var ( var ( goroot string releaseRegexp = regexp.MustCompile(`^(release|weekly)\.[0-9\-.]+`) - benchRequests vector.Vector ) func main() { @@ -77,7 +71,7 @@ func main() { os.Exit(2) } flag.Parse() - if len(flag.Args()) == 0 { + if len(flag.Args()) == 0 && !*commitFlag { flag.Usage() } goroot = path.Join(*buildroot, "goroot") @@ -101,17 +95,24 @@ func main() { log.Fatal("Error cloning repository:", err) } + if *commitFlag { + if len(flag.Args()) == 0 { + commitWatcher() + return + } + go commitWatcher() + } + // if specified, build revision and return if *buildRevision != "" { - c, err := getCommit(*buildRevision) + hash, err := fullHash(*buildRevision) if err != nil { log.Fatal("Error finding revision: ", err) } for _, b := range builders { - if err := b.buildCommit(c); err != nil { + if err := b.buildHash(hash); err != nil { log.Println(err) } - runQueuedBenchmark() } return } @@ -127,13 +128,8 @@ func main() { // go continuous build mode (default) // check for new commits and build them for { - err := run(nil, goroot, "hg", "pull", "-u") - if err != nil { - log.Println("hg pull failed:", err) - time.Sleep(waitInterval) - continue - } built := false + t := time.Nanoseconds() if *parallel { done := make(chan bool) for _, b := range builders { @@ -149,46 +145,15 @@ func main() { built = b.build() || built } } - // only run benchmarks if we didn't build anything - // so that they don't hold up the builder queue + // sleep if there was nothing to build if !built { - if !runQueuedBenchmark() { - // if we have no benchmarks to do, pause - time.Sleep(waitInterval) - } - // after running one benchmark, - // continue to find and build new revisions. + time.Sleep(waitInterval) + } + // sleep if we're looping too fast. + t1 := time.Nanoseconds() - t + if t1 < waitInterval { + time.Sleep(waitInterval - t1) } - } -} - -func runQueuedBenchmark() bool { - if benchRequests.Len() == 0 { - return false - } - runBenchmark(benchRequests.Pop().(BenchRequest)) - return true -} - -func runBenchmark(r BenchRequest) { - // run benchmarks and send to dashboard - log.Println(r.builder.name, "benchmarking", r.commit.num) - defer os.RemoveAll(r.path) - pkg := path.Join(r.path, "go", "src", "pkg") - bin := path.Join(r.path, "go", "bin") - env := []string{ - "GOOS=" + r.builder.goos, - "GOARCH=" + r.builder.goarch, - "PATH=" + bin + ":" + os.Getenv("PATH"), - } - logfile := path.Join(r.path, "bench.log") - benchLog, _, err := runLog(env, logfile, pkg, "gomake", "bench") - if err != nil { - log.Println(r.builder.name, "gomake bench:", err) - return - } - if err = r.builder.recordBenchmarks(benchLog, r.commit); err != nil { - log.Println("recordBenchmarks:", err) } } @@ -235,7 +200,7 @@ func (b *Builder) buildExternal() { log.Println("hg pull failed:", err) continue } - c, tag, err := getTag(releaseRegexp) + hash, tag, err := firstTag(releaseRegexp) if err != nil { log.Println(err) continue @@ -249,8 +214,8 @@ func (b *Builder) buildExternal() { if tag == prevTag && time.Nanoseconds() < nextBuild { continue } - // buildCommit will also build the packages - if err := b.buildCommit(c); err != nil { + // build will also build the packages + if err := b.buildHash(hash); err != nil { log.Println(err) continue } @@ -269,65 +234,46 @@ func (b *Builder) build() bool { log.Println(b.name, "build:", err) } }() - c, err := b.nextCommit() + hash, err := b.todo() if err != nil { log.Println(err) return false } - if c == nil { + if hash == "" { return false } - err = b.buildCommit(*c) - if err != nil { - log.Println(err) - } - return true -} + // Look for hash locally before running hg pull. -// nextCommit returns the next unbuilt Commit for this builder -func (b *Builder) nextCommit() (nextC *Commit, err os.Error) { - defer func() { - if err != nil { - err = fmt.Errorf("%s nextCommit: %s", b.name, err) + if _, err := fullHash(hash[:12]); err != nil { + // Don't have hash, so run hg pull. + if err := run(nil, goroot, "hg", "pull"); err != nil { + log.Println("hg pull failed:", err) + return false } - }() - hw, err := b.getHighWater() - if err != nil { - return } - c, err := getCommit(hw) + err = b.buildHash(hash) if err != nil { - return - } - next := c.num + 1 - c, err = getCommit(strconv.Itoa(next)) - if err == nil && c.num == next { - return &c, nil + log.Println(err) } - return nil, nil + return true } -func (b *Builder) buildCommit(c Commit) (err os.Error) { +func (b *Builder) buildHash(hash string) (err os.Error) { defer func() { if err != nil { - err = fmt.Errorf("%s buildCommit: %d: %s", b.name, c.num, err) + err = fmt.Errorf("%s build: %s: %s", b.name, hash, err) } }() - log.Println(b.name, "building", c.num) + log.Println(b.name, "building", hash) // create place in which to do work - workpath := path.Join(*buildroot, b.name+"-"+strconv.Itoa(c.num)) + workpath := path.Join(*buildroot, b.name+"-"+hash[:12]) err = os.Mkdir(workpath, mkdirPerm) if err != nil { return } - benchRequested := false - defer func() { - if !benchRequested { - os.RemoveAll(workpath) - } - }() + defer os.RemoveAll(workpath) // clone repo err = run(nil, workpath, "hg", "clone", goroot, "go") @@ -337,7 +283,7 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { // update to specified revision err = run(nil, path.Join(workpath, "go"), - "hg", "update", "-r", strconv.Itoa(c.num)) + "hg", "update", hash) if err != nil { return } @@ -348,7 +294,7 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { logfile := path.Join(workpath, "build.log") buildLog, status, err := runLog(b.envv(), logfile, srcDir, *buildCmd) if err != nil { - return fmt.Errorf("all.bash: %s", err) + return fmt.Errorf("%s: %s", *buildCmd, err) } // if we're in external mode, build all packages and return @@ -356,36 +302,27 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { if status != 0 { return os.NewError("go build failed") } - return b.buildPackages(workpath, c) + return b.buildPackages(workpath, hash) } if status != 0 { // record failure - return b.recordResult(buildLog, c) + return b.recordResult(buildLog, hash) } // record success - if err = b.recordResult("", c); err != nil { + if err = b.recordResult("", hash); err != nil { return fmt.Errorf("recordResult: %s", err) } - // send benchmark request if benchmarks are enabled - if *runBenchmarks { - benchRequests.Insert(0, BenchRequest{ - builder: b, - commit: c, - path: workpath, - }) - benchRequested = true - } - // finish here if codeUsername and codePassword aren't set if b.codeUsername == "" || b.codePassword == "" || !*buildRelease { return } // if this is a release, create tgz and upload to google code - if release := releaseRegexp.FindString(c.desc); release != "" { + releaseHash, release, err := firstTag(releaseRegexp) + if hash == releaseHash { // clean out build state err = run(b.envv(), srcDir, "./clean.bash", "--nopkg") if err != nil { @@ -411,6 +348,9 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { // envv returns an environment for build/bench execution func (b *Builder) envv() []string { + if runtime.GOOS == "windows" { + return b.envvWindows() + } e := []string{ "GOOS=" + b.goos, "GOARCH=" + b.goarch, @@ -422,6 +362,42 @@ func (b *Builder) envv() []string { return e } +// windows version of envv +func (b *Builder) envvWindows() []string { + start := map[string]string{ + "GOOS": b.goos, + "GOARCH": b.goarch, + "GOROOT_FINAL": "/c/go", + } + for _, name := range extraEnv { + start[name] = os.Getenv(name) + } + skip := map[string]bool{ + "GOBIN": true, + "GOROOT": true, + "INCLUDE": true, + "LIB": true, + } + var e []string + for name, v := range start { + e = append(e, name+"="+v) + skip[name] = true + } + for _, kv := range os.Environ() { + s := strings.Split(kv, "=", 2) + name := strings.ToUpper(s[0]) + switch { + case name == "": + // variables, like "=C:=C:\", just copy them + e = append(e, kv) + case !skip[name]: + e = append(e, kv) + skip[name] = true + } + } + return e +} + func isDirectory(name string) bool { s, err := os.Stat(name) return err == nil && s.IsDirectory() @@ -431,3 +407,209 @@ func isFile(name string) bool { s, err := os.Stat(name) return err == nil && (s.IsRegular() || s.IsSymlink()) } + +// commitWatcher polls hg for new commits and tells the dashboard about them. +func commitWatcher() { + // Create builder just to get master key. + b, err := NewBuilder("mercurial-commit") + if err != nil { + log.Fatal(err) + } + for { + if *verbose { + log.Printf("poll...") + } + commitPoll(b.key) + if *verbose { + log.Printf("sleep...") + } + time.Sleep(60e9) + } +} + +// HgLog represents a single Mercurial revision. +type HgLog struct { + Hash string + Author string + Date string + Desc string + Parent string + + // Internal metadata + added bool +} + +// logByHash is a cache of all Mercurial revisions we know about, +// indexed by full hash. +var logByHash = map[string]*HgLog{} + +// xmlLogTemplate is a template to pass to Mercurial to make +// hg log print the log in valid XML for parsing with xml.Unmarshal. +const xmlLogTemplate = ` + <log> + <hash>{node|escape}</hash> + <parent>{parent|escape}</parent> + <author>{author|escape}</author> + <date>{date}</date> + <desc>{desc|escape}</desc> + </log> +` + +// commitPoll pulls any new revisions from the hg server +// and tells the server about them. +func commitPoll(key string) { + // Catch unexpected panics. + defer func() { + if err := recover(); err != nil { + log.Printf("commitPoll panic: %s", err) + } + }() + + if err := run(nil, goroot, "hg", "pull"); err != nil { + log.Printf("hg pull: %v", err) + return + } + + const N = 20 // how many revisions to grab + + data, _, err := runLog(nil, "", goroot, "hg", "log", + "--encoding=utf-8", + "--limit="+strconv.Itoa(N), + "--template="+xmlLogTemplate, + ) + if err != nil { + log.Printf("hg log: %v", err) + return + } + + var logStruct struct { + Log []HgLog + } + err = xml.Unmarshal(strings.NewReader("<top>"+data+"</top>"), &logStruct) + if err != nil { + log.Printf("unmarshal hg log: %v", err) + return + } + + logs := logStruct.Log + + // Pass 1. Fill in parents and add new log entries to logsByHash. + // Empty parent means take parent from next log entry. + // Non-empty parent has form 1234:hashhashhash; we want full hash. + for i := range logs { + l := &logs[i] + log.Printf("hg log: %s < %s\n", l.Hash, l.Parent) + if l.Parent == "" && i+1 < len(logs) { + l.Parent = logs[i+1].Hash + } else if l.Parent != "" { + l.Parent, _ = fullHash(l.Parent) + } + if l.Parent == "" { + // Can't create node without parent. + continue + } + + if logByHash[l.Hash] == nil { + // Make copy to avoid pinning entire slice when only one entry is new. + t := *l + logByHash[t.Hash] = &t + } + } + + for i := range logs { + l := &logs[i] + if l.Parent == "" { + continue + } + addCommit(l.Hash, key) + } +} + +// addCommit adds the commit with the named hash to the dashboard. +// key is the secret key for authentication to the dashboard. +// It avoids duplicate effort. +func addCommit(hash, key string) bool { + l := logByHash[hash] + if l == nil { + return false + } + if l.added { + return true + } + + // Check for already added, perhaps in an earlier run. + if dashboardCommit(hash) { + log.Printf("%s already on dashboard\n", hash) + // Record that this hash is on the dashboard, + // as must be all its parents. + for l != nil { + l.added = true + l = logByHash[l.Parent] + } + return true + } + + // Create parent first, to maintain some semblance of order. + if !addCommit(l.Parent, key) { + return false + } + + // Create commit. + if err := postCommit(key, l); err != nil { + log.Printf("failed to add %s to dashboard: %v", key, err) + return false + } + return true +} + +// fullHash returns the full hash for the given Mercurial revision. +func fullHash(rev string) (hash string, err os.Error) { + defer func() { + if err != nil { + err = fmt.Errorf("fullHash: %s: %s", rev, err) + } + }() + s, _, err := runLog(nil, "", goroot, + "hg", "log", + "--encoding=utf-8", + "--rev="+rev, + "--limit=1", + "--template={node}", + ) + if err != nil { + return + } + s = strings.TrimSpace(s) + if s == "" { + return "", fmt.Errorf("cannot find revision") + } + if len(s) != 20 { + return "", fmt.Errorf("hg returned invalid hash " + s) + } + return s, nil +} + +var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`) + +// firstTag returns the hash and tag of the most recent tag matching re. +func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) { + o, _, err := runLog(nil, "", goroot, "hg", "tags") + for _, l := range strings.Split(o, "\n", -1) { + if l == "" { + continue + } + s := revisionRe.FindStringSubmatch(l) + if s == nil { + err = os.NewError("couldn't find revision number") + return + } + if !re.MatchString(s[1]) { + continue + } + tag = s[1] + hash, err = fullHash(s[3]) + return + } + err = os.NewError("no matching tag found") + return +} diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go index 6e9f9ff39..ee65d7669 100644 --- a/misc/dashboard/builder/package.go +++ b/misc/dashboard/builder/package.go @@ -13,8 +13,8 @@ import ( "path" ) -func (b *Builder) buildPackages(workpath string, c Commit) os.Error { - pkgs, err := getPackages() +func (b *Builder) buildPackages(workpath string, hash string) os.Error { + pkgs, err := packages() if err != nil { return err } @@ -32,13 +32,13 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error { built := code != 0 // get doc comment from package source - info, err := getPackageComment(p, path.Join(goroot, "pkg", p)) + info, err := packageComment(p, path.Join(goroot, "pkg", p)) if err != nil { log.Printf("goinstall %v: %v", p, err) } // update dashboard with build state + info - err = b.updatePackage(p, built, buildLog, info, c) + err = b.updatePackage(p, built, buildLog, info, hash) if err != nil { log.Printf("updatePackage %v: %v", p, err) } @@ -46,7 +46,7 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error { return nil } -func getPackageComment(pkg, pkgpath string) (info string, err os.Error) { +func packageComment(pkg, pkgpath string) (info string, err os.Error) { fset := token.NewFileSet() pkgs, err := parser.ParseDir(fset, pkgpath, nil, parser.PackageClauseOnly|parser.ParseComments) if err != nil { diff --git a/misc/dashboard/godashboard/app.yaml b/misc/dashboard/godashboard/app.yaml index aec559dcc..83611cf90 100644 --- a/misc/dashboard/godashboard/app.yaml +++ b/misc/dashboard/godashboard/app.yaml @@ -1,9 +1,13 @@ application: godashboard -version: 5 +version: 7 runtime: python api_version: 1 handlers: +- url: /favicon\.ico + static_files: static/favicon.ico + upload: static/favicon\.ico + - url: /static static_dir: static diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py index 035bf842f..5678f2e1b 100644 --- a/misc/dashboard/godashboard/gobuild.py +++ b/misc/dashboard/godashboard/gobuild.py @@ -5,21 +5,19 @@ # This is the server part of the continuous build system for Go. It must be run # by AppEngine. +from django.utils import simplejson from google.appengine.api import mail from google.appengine.api import memcache from google.appengine.ext import db from google.appengine.ext import webapp from google.appengine.ext.webapp import template from google.appengine.ext.webapp.util import run_wsgi_app -import binascii import datetime import hashlib import hmac import logging import os import re -import struct -import time import bz2 # local imports @@ -50,10 +48,6 @@ class Commit(db.Model): fail_notification_sent = db.BooleanProperty() -class Cache(db.Model): - data = db.BlobProperty() - expire = db.IntegerProperty() - # A CompressedLog contains the textual build log of a failed build. # The key name is the hex digest of the SHA256 hash of the contents. # The contents is bz2 compressed. @@ -62,23 +56,6 @@ class CompressedLog(db.Model): N = 30 -def cache_get(key): - c = Cache.get_by_key_name(key) - if c is None or c.expire < time.time(): - return None - return c.data - -def cache_set(key, val, timeout): - c = Cache(key_name = key) - c.data = val - c.expire = int(time.time() + timeout) - c.put() - -def cache_del(key): - c = Cache.get_by_key_name(key) - if c is not None: - c.delete() - def builderInfo(b): f = b.split('-', 3) goos = f[0] @@ -96,7 +73,7 @@ def builderset(): for c in results: builders.update(set(parseBuild(build)['builder'] for build in c.builds)) return builders - + class MainPage(webapp.RequestHandler): def get(self): self.response.headers['Content-Type'] = 'text/html; charset=utf-8' @@ -147,7 +124,30 @@ class MainPage(webapp.RequestHandler): path = os.path.join(os.path.dirname(__file__), 'main.html') self.response.out.write(template.render(path, values)) -class GetHighwater(webapp.RequestHandler): +# A DashboardHandler is a webapp.RequestHandler but provides +# authenticated_post - called by post after authenticating +# json - writes object in json format to response output +class DashboardHandler(webapp.RequestHandler): + def post(self): + if not auth(self.request): + self.response.set_status(403) + return + self.authenticated_post() + + def authenticated_post(self): + return + + def json(self, obj): + self.response.set_status(200) + simplejson.dump(obj, self.response.out) + return + +def auth(req): + k = req.get('key') + return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey + +# Todo serves /todo. It tells the builder which commits need to be built. +class Todo(DashboardHandler): def get(self): builder = self.request.get('builder') key = 'todo-%s' % builder @@ -155,28 +155,19 @@ class GetHighwater(webapp.RequestHandler): if response is None: # Fell out of memcache. Rebuild from datastore results. # We walk the commit list looking for nodes that have not - # been built by this builder and record the *parents* of those - # nodes, because each builder builds the revision *after* the - # one return (because we might not know about the latest - # revision). + # been built by this builder. q = Commit.all() q.order('-__key__') todo = [] - need = False first = None for c in q.fetch(N+1): if first is None: first = c - if need: - todo.append(c.node) - need = not built(c, builder) - if not todo: - todo.append(first.node) - response = ' '.join(todo) + if not built(c, builder): + todo.append({'Hash': c.node}) + response = simplejson.dumps(todo) memcache.set(key, response, 3600) self.response.set_status(200) - if self.request.get('all') != 'yes': - response = response.split()[0] self.response.out.write(response) def built(c, builder): @@ -185,22 +176,8 @@ def built(c, builder): return True return False -def auth(req): - k = req.get('key') - return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey - -class SetHighwater(webapp.RequestHandler): - def post(self): - if not auth(self.request): - self.response.set_status(403) - return - - # Allow for old builders. - # This is a no-op now: we figure out what to build based - # on the current dashboard status. - return - -class LogHandler(webapp.RequestHandler): +# Log serves /log/. It retrieves log data by content hash. +class LogHandler(DashboardHandler): def get(self): self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' hash = self.request.path[5:] @@ -214,12 +191,8 @@ class LogHandler(webapp.RequestHandler): # Init creates the commit with id 0. Since this commit doesn't have a parent, # it cannot be created by Build. -class Init(webapp.RequestHandler): - def post(self): - if not auth(self.request): - self.response.set_status(403) - return - +class Init(DashboardHandler): + def authenticated_post(self): date = parseDate(self.request.get('date')) node = self.request.get('node') if not validNode(node) or date is None: @@ -239,7 +212,86 @@ class Init(webapp.RequestHandler): self.response.set_status(200) -# Build is the main command: it records the result of a new build. +# The last commit when we switched to using entity groups. +# This is the root of the new commit entity group. +RootCommitKeyName = '00000f26-f32c6f1038207c55d5780231f7484f311020747e' + +# CommitHandler serves /commit. +# A GET of /commit retrieves information about the specified commit. +# A POST of /commit creates a node for the given commit. +# If the commit already exists, the POST silently succeeds (like mkdir -p). +class CommitHandler(DashboardHandler): + def get(self): + node = self.request.get('node') + if not validNode(node): + return self.json({'Status': 'FAIL', 'Error': 'malformed node hash'}) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown revision'}) + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + + def authenticated_post(self): + # Require auth with the master key, not a per-builder key. + if self.request.get('builder'): + self.response.set_status(403) + return + + node = self.request.get('node') + date = parseDate(self.request.get('date')) + user = self.request.get('user').encode('utf8') + desc = self.request.get('desc').encode('utf8') + parenthash = self.request.get('parent') + + if not validNode(node) or not validNode(parenthash) or date is None: + return self.json({'Status': 'FAIL', 'Error': 'malformed node, parent, or date'}) + + n = nodeByHash(node) + if n is None: + p = nodeByHash(parenthash) + if p is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown parent'}) + + # Want to create new node in a transaction so that multiple + # requests creating it do not collide and so that multiple requests + # creating different nodes get different sequence numbers. + # All queries within a transaction must include an ancestor, + # but the original datastore objects we used for the dashboard + # have no common ancestor. Instead, we use a well-known + # root node - the last one before we switched to entity groups - + # as the as the common ancestor. + root = Commit.get_by_key_name(RootCommitKeyName) + + def add_commit(): + if nodeByHash(node, ancestor=root) is not None: + return + + # Determine number for this commit. + # Once we have created one new entry it will be lastRooted.num+1, + # but the very first commit created in this scheme will have to use + # last.num's number instead (last is likely not rooted). + q = Commit.all() + q.order('-__key__') + q.ancestor(root) + last = q.fetch(1)[0] + num = last.num+1 + + n = Commit(key_name = '%08x-%s' % (num, node), parent = root) + n.num = num + n.node = node + n.parentnode = parenthash + n.user = user + n.date = date + n.desc = desc + n.put() + db.run_in_transaction(add_commit) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'failed to create commit node'}) + + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + +# Build serves /build. +# A POST to /build records a new build result. class Build(webapp.RequestHandler): def post(self): if not auth(self.request): @@ -256,44 +308,33 @@ class Build(webapp.RequestHandler): l.log = bz2.compress(log) l.put() - date = parseDate(self.request.get('date')) - user = self.request.get('user').encode('utf8') - desc = self.request.get('desc').encode('utf8') node = self.request.get('node') - parenthash = self.request.get('parent') - if not validNode(node) or not validNode(parenthash) or date is None: - logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date')) + if not validNode(node): + logging.error('Invalid node %s' % (node)) self.response.set_status(500) return - q = Commit.all() - q.filter('node =', parenthash) - parent = q.get() - if parent is None: - logging.error('Cannot find parent %s of node %s' % (parenthash, node)) + n = nodeByHash(node) + if n is None: + logging.error('Cannot find node %s' % (node)) self.response.set_status(404) return - parentnum, _ = parent.key().name().split('-', 1) - nodenum = int(parentnum, 16) + 1 - - key_name = '%08x-%s' % (nodenum, node) + nn = n def add_build(): - n = Commit.get_by_key_name(key_name) + n = nodeByHash(node, ancestor=nn) if n is None: - n = Commit(key_name = key_name) - n.num = nodenum - n.node = node - n.parentnode = parenthash - n.user = user - n.date = date - n.desc = desc + logging.error('Cannot find hash in add_build: %s %s' % (builder, node)) + return + s = '%s`%s' % (builder, loghash) for i, b in enumerate(n.builds): if b.split('`', 1)[0] == builder: + # logging.error('Found result for %s %s already' % (builder, node)) n.builds[i] = s break else: + # logging.error('Added result for %s %s' % (builder, node)) n.builds.append(s) n.put() @@ -302,39 +343,101 @@ class Build(webapp.RequestHandler): key = 'todo-%s' % builder memcache.delete(key) - def mark_sent(): - n = Commit.get_by_key_name(key_name) - n.fail_notification_sent = True - n.put() - - n = Commit.get_by_key_name(key_name) - if loghash and not failed(parent, builder) and not n.fail_notification_sent: - subject = const.mail_fail_subject % (builder, desc.split("\n")[0]) - path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt') - body = template.render(path, { - "builder": builder, - "node": node[:12], - "user": user, - "desc": desc, - "loghash": loghash - }) - mail.send_mail( - sender=const.mail_from, - reply_to=const.mail_fail_reply_to, - to=const.mail_fail_to, - subject=subject, - body=body - ) - db.run_in_transaction(mark_sent) + c = getBrokenCommit(node, builder) + if c is not None and not c.fail_notification_sent: + notifyBroken(c, builder) self.response.set_status(200) -def failed(c, builder): + +def getBrokenCommit(node, builder): + """ + getBrokenCommit returns a Commit that breaks the build. + The Commit will be either the one specified by node or the one after. + """ + + # Squelch mail if already fixed. + head = firstResult(builder) + if broken(head, builder) == False: + return + + # Get current node and node before, after. + cur = nodeByHash(node) + if cur is None: + return + before = nodeBefore(cur) + after = nodeAfter(cur) + + if broken(before, builder) == False and broken(cur, builder): + return cur + if broken(cur, builder) == False and broken(after, builder): + return after + + return + +def firstResult(builder): + q = Commit.all().order('-__key__') + for c in q.fetch(20): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return c + return None + +def nodeBefore(c): + return nodeByHash(c.parentnode) + +def nodeAfter(c): + return Commit.all().filter('parenthash', c.node).get() + +def notifyBroken(c, builder): + def send(): + n = Commit.get(c.key()) + if n is None: + logging.error("couldn't retrieve Commit '%s'" % c.key()) + return False + if n.fail_notification_sent: + return False + n.fail_notification_sent = True + return n.put() + if not db.run_in_transaction(send): + return + + subject = const.mail_fail_subject % (builder, c.desc.split('\n')[0]) + path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt') + body = template.render(path, { + "builder": builder, + "node": c.node, + "user": c.user, + "desc": c.desc, + "loghash": logHash(c, builder) + }) + mail.send_mail( + sender=const.mail_from, + to=const.mail_fail_to, + subject=subject, + body=body + ) + +def logHash(c, builder): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return p[1] + return "" + +def broken(c, builder): + """ + broken returns True if commit c breaks the build for the specified builder, + False if it is a good build, and None if no results exist for this builder. + """ + if c is None: + return None for i, b in enumerate(c.builds): p = b.split('`', 1) if p[0] == builder: return len(p[1]) > 0 - return False + return None def node(num): q = Commit.all() @@ -342,6 +445,24 @@ def node(num): n = q.get() return n +def nodeByHash(hash, ancestor=None): + q = Commit.all() + q.filter('node =', hash) + if ancestor is not None: + q.ancestor(ancestor) + n = q.get() + return n + +# nodeObj returns a JSON object (ready to be passed to simplejson.dump) describing node. +def nodeObj(n): + return { + 'Hash': n.node, + 'ParentHash': n.parentnode, + 'User': n.user, + 'Date': n.date.strftime('%Y-%m-%d %H:%M %z'), + 'Desc': n.desc, + } + class FixedOffset(datetime.tzinfo): """Fixed offset in minutes east from UTC.""" @@ -417,15 +538,20 @@ def toRev(c): def byBuilder(x, y): return cmp(x['builder'], y['builder']) +# Give old builders work; otherwise they pound on the web site. +class Hwget(DashboardHandler): + def get(self): + self.response.out.write("8000\n") + # This is the URL map for the server. The first three entries are public, the # rest are only used by the builders. application = webapp.WSGIApplication( [('/', MainPage), + ('/hw-get', Hwget), ('/log/.*', LogHandler), - ('/hw-get', GetHighwater), - ('/hw-set', SetHighwater), - + ('/commit', CommitHandler), ('/init', Init), + ('/todo', Todo), ('/build', Build), ], debug=True) diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml index 148824bb6..4a00c4a6f 100644 --- a/misc/dashboard/godashboard/index.yaml +++ b/misc/dashboard/godashboard/index.yaml @@ -23,6 +23,12 @@ indexes: - name: __key__ direction: desc +- kind: Commit + ancestor: yes + properties: + - name: __key__ + direction: desc + - kind: Project properties: - name: approved diff --git a/misc/dashboard/godashboard/static/favicon.ico b/misc/dashboard/godashboard/static/favicon.ico Binary files differnew file mode 100644 index 000000000..48854ff3b --- /dev/null +++ b/misc/dashboard/godashboard/static/favicon.ico diff --git a/misc/dashboard/googlecode_upload.py b/misc/dashboard/googlecode_upload.py index 3b1d432ff..e87db884a 100755 --- a/misc/dashboard/googlecode_upload.py +++ b/misc/dashboard/googlecode_upload.py @@ -70,7 +70,7 @@ def upload(file, project_name, user_name, password, summary, labels=None): Returns: a tuple: http_status: 201 if the upload succeeded, something else if an - error occured. + error occurred. http_reason: The human-readable string associated with http_status file_url: If the upload succeeded, the URL of the file on Google Code, None otherwise. diff --git a/misc/emacs/go-mode-load.el b/misc/emacs/go-mode-load.el index 0ace46dfa..d453166a4 100644 --- a/misc/emacs/go-mode-load.el +++ b/misc/emacs/go-mode-load.el @@ -19,7 +19,7 @@ ;;;### (autoloads (gofmt-before-save gofmt go-mode) "go-mode" "go-mode.el" -;;;;;; (19847 61431)) +;;;;;; (19917 17808)) ;;; Generated autoloads from go-mode.el (autoload 'go-mode "go-mode" "\ diff --git a/misc/emacs/go-mode.el b/misc/emacs/go-mode.el index 692cabfe5..03f0a2a8b 100644 --- a/misc/emacs/go-mode.el +++ b/misc/emacs/go-mode.el @@ -507,7 +507,9 @@ Replace the current buffer on success; display errors on failure." (let ((srcbuf (current-buffer))) (with-temp-buffer (let ((outbuf (current-buffer)) - (errbuf (get-buffer-create "*Gofmt Errors*"))) + (errbuf (get-buffer-create "*Gofmt Errors*")) + (coding-system-for-read 'utf-8) ;; use utf-8 with subprocesses + (coding-system-for-write 'utf-8)) (with-current-buffer errbuf (erase-buffer)) (with-current-buffer srcbuf (save-restriction @@ -521,7 +523,7 @@ Replace the current buffer on success; display errors on failure." (erase-buffer) (insert-buffer-substring outbuf) (goto-char (min old-point (point-max))) - (if old-mark (set-mark (min old-mark (point-max)))) + (if old-mark (push-mark (min old-mark (point-max)) t)) (kill-buffer errbuf)) ;; gofmt failed: display the errors diff --git a/misc/godoc/README b/misc/godoc/README new file mode 100644 index 000000000..3c8d830e4 --- /dev/null +++ b/misc/godoc/README @@ -0,0 +1,22 @@ +Instructions to get an initial godoc running on a local app engine emulator +--------------------------------------------------------------------------- + +To run godoc under the app engine emulator, create a ("goroot") godoc +directory that contains the app.yaml file, the doc and lib directories +from the Go distribution, as well as a godoc directory with the godoc +sources from src/cmd/godoc. In the godoc source directory, replace +main.go with init.go. The directory structure should look as follows: + +godoc // "goroot" directory + app.yaml // app engine control file + doc // goroot/doc directory + favicon.ico + godoc // contains godoc sources + godoc.go // unchanged godoc file + init.go // this file instead of godoc/main.go + ... // remaining godoc files + lib // goroot/lib directory + +Run app engine emulator locally: dev_appserver.py -a <hostname> godoc +where godoc is the top-level "goroot" directory. The godoc home page +is then served at: <hostname>:8080 . diff --git a/misc/godoc/app.yaml b/misc/godoc/app.yaml new file mode 100644 index 000000000..f8b46db31 --- /dev/null +++ b/misc/godoc/app.yaml @@ -0,0 +1,12 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +application: godoc +version: 1 +runtime: go +api_version: 1 + +handlers: +- url: /.* + script: _go_app diff --git a/misc/godoc/init.go b/misc/godoc/init.go new file mode 100644 index 000000000..0fd0bd542 --- /dev/null +++ b/misc/godoc/init.go @@ -0,0 +1,35 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file replaces main.go when running godoc under the app engine emulator. +// See the README file for instructions. + +package main + +import ( + "http" + "log" + "os" + "path/filepath" +) + +func serveError(w http.ResponseWriter, r *http.Request, relpath string, err os.Error) { + contents := applyTemplate(errorHTML, "errorHTML", err) // err may contain an absolute path! + w.WriteHeader(http.StatusNotFound) + servePage(w, "File "+relpath, "", "", contents) +} + +func init() { + // set goroot + cwd, err := os.Getwd() + if err != nil { + log.Fatalf("cwd: %s", err) + } + log.Printf("cwd = %s", cwd) + *goroot = filepath.Clean(cwd) + + initHandlers() + readTemplates() + registerPublicHandlers(http.DefaultServeMux) +} diff --git a/misc/goplay/goplay.go b/misc/goplay/goplay.go index f3e2ff565..f1dc1bca5 100644 --- a/misc/goplay/goplay.go +++ b/misc/goplay/goplay.go @@ -5,7 +5,6 @@ package main import ( - "bytes" "exec" "flag" "http" @@ -140,32 +139,7 @@ func error(w http.ResponseWriter, out []byte, err os.Error) { // run executes the specified command and returns its output and an error. func run(cmd ...string) ([]byte, os.Error) { - // find the specified binary - bin, err := exec.LookPath(cmd[0]) - if err != nil { - // report binary as well as the error - return nil, os.NewError(cmd[0] + ": " + err.String()) - } - - // run the binary and read its combined stdout and stderr into a buffer - p, err := exec.Run(bin, cmd, os.Environ(), "", exec.DevNull, exec.Pipe, exec.MergeWithStdout) - if err != nil { - return nil, err - } - var buf bytes.Buffer - io.Copy(&buf, p.Stdout) - w, err := p.Wait(0) - p.Close() - if err != nil { - return nil, err - } - - // set the error return value if the program had a non-zero exit status - if !w.Exited() || w.ExitStatus() != 0 { - err = os.ErrorString("running " + cmd[0] + ": " + w.String()) - } - - return buf.Bytes(), err + return exec.Command(cmd[0], cmd[1:]...).CombinedOutput() } var frontPage, output *template.Template // HTML templates diff --git a/misc/vim/ftplugin/go/fmt.vim b/misc/vim/ftplugin/go/fmt.vim index 18a2156f5..a299dfcee 100644 --- a/misc/vim/ftplugin/go/fmt.vim +++ b/misc/vim/ftplugin/go/fmt.vim @@ -13,7 +13,7 @@ " replacing the buffer with stderr output. " -command! Fmt call s:GoFormat() +command! -buffer Fmt call s:GoFormat() function! s:GoFormat() let view = winsaveview() diff --git a/misc/vim/indent/go.vim b/misc/vim/indent/go.vim index 2e9f191f5..faf4d79e2 100644 --- a/misc/vim/indent/go.vim +++ b/misc/vim/indent/go.vim @@ -4,27 +4,62 @@ " " indent/go.vim: Vim indent file for Go. " +" TODO: +" - function invocations split across lines +" - general line splits (line ends in an operator) if exists("b:did_indent") finish endif let b:did_indent = 1 -" C indentation is mostly correct -setlocal cindent - -" Options set: -" +0 -- Don't indent continuation lines (because Go doesn't use semicolons -" much) -" L0 -- Don't move jump labels (NOTE: this isn't correct when working with -" gofmt, but it does keep struct literals properly indented.) -" :0 -- Align case labels with switch statement -" l1 -- Always align case body relative to case labels -" J1 -- Indent JSON-style objects (properly indents struct-literals) -" (0, Ws -- Indent lines inside of unclosed parentheses by one shiftwidth -" m1 -- Align closing parenthesis line with first non-blank of matching -" parenthesis line -" -" Known issue: Trying to do a multi-line struct literal in a short variable -" declaration will not indent properly. -setlocal cinoptions+=+0,L0,:0,l1,J1,(0,Ws,m1 +" C indentation is too far off useful, mainly due to Go's := operator. +" Let's just define our own. +setlocal nolisp +setlocal autoindent +setlocal indentexpr=GoIndent(v:lnum) +setlocal indentkeys+=<:>,0=},0=) + +if exists("*GoIndent") + finish +endif + +function! GoIndent(lnum) + let prevlnum = prevnonblank(a:lnum-1) + if prevlnum == 0 + " top of file + return 0 + endif + + " grab the previous and current line, stripping comments. + let prevl = substitute(getline(prevlnum), '//.*$', '', '') + let thisl = substitute(getline(a:lnum), '//.*$', '', '') + let previ = indent(prevlnum) + + let ind = previ + + if prevl =~ '[({]\s*$' + " previous line opened a block + let ind += &sw + endif + if prevl =~# '^\s*\(case .*\|default\):$' + " previous line is part of a switch statement + let ind += &sw + endif + " TODO: handle if the previous line is a label. + + if thisl =~ '^\s*[)}]' + " this line closed a block + let ind -= &sw + endif + + " Colons are tricky. + " We want to outdent if it's part of a switch ("case foo:" or "default:"). + " We ignore trying to deal with jump labels because (a) they're rare, and + " (b) they're hard to disambiguate from a composite literal key. + if thisl =~# '^\s*\(case .*\|default\):$' + let ind -= &sw + endif + + return ind +endfunction diff --git a/src/Make.ccmd b/src/Make.ccmd index cb2b25512..88f647152 100644 --- a/src/Make.ccmd +++ b/src/Make.ccmd @@ -18,6 +18,9 @@ CLEANFILES+=y.tab.[ch] clean: rm -f *.$(HOST_O) $(TARG) $(CLEANFILES) +nuke: clean + rm -f "$(GOBIN)/$(TARG)" + ifneq ($(NOINSTALL),1) install: $(QUOTED_GOBIN)/$(TARG) endif diff --git a/src/Make.clib b/src/Make.clib index ebe4f84b9..25fe88463 100644 --- a/src/Make.clib +++ b/src/Make.clib @@ -26,6 +26,8 @@ CLEANFILES+=y.tab.[ch] y.output a.out $(LIB) clean: rm -f *.$(HOST_O) $(CLEANFILES) +nuke: clean + rm -f "$(GOROOT)/lib/$(LIB)" y.tab.h: $(YFILES) LANG=C LANGUAGE="en_US.UTF8" bison -v -y $(HOST_YFLAGS) $(YFILES) diff --git a/src/Make.cmd b/src/Make.cmd index 26c3ca2fc..27c6a2e13 100644 --- a/src/Make.cmd +++ b/src/Make.cmd @@ -27,7 +27,7 @@ install: $(TARGDIR)/$(TARG) $(TARGDIR)/$(TARG): $(TARG) mkdir -p $(TARGDIR) && cp -f $(TARG) $(TARGDIR) -CLEANFILES+=$(TARG) _test _testmain.go +CLEANFILES+=$(TARG) _test _testmain.go test.out build.out nuke: clean rm -f $(TARGDIR)/$(TARG) diff --git a/src/Make.pkg b/src/Make.pkg index 966bc61c7..86a2e9fd0 100644 --- a/src/Make.pkg +++ b/src/Make.pkg @@ -79,7 +79,7 @@ testpackage-clean: install: $(INSTALLFILES) $(TARGDIR)/$(TARG).a: _obj/$(TARG).a - @test -d $(QUOTED_GOROOT)/pkg && mkdir -p $(TARGDIR)/$(dir) + @mkdir -p $(TARGDIR)/$(dir) cp _obj/$(TARG).a "$@" _go_.$O: $(GOFILES) $(PREREQ) @@ -227,7 +227,7 @@ _obj/$(SWIG_PREFIX)-%.so: _obj/%_wrapcxx.o $(HOST_CXX) $(_CGO_CFLAGS_$(GOARCH)) -o $@ $^ $(SWIG_LDFLAGS) $(_CGO_LDFLAGS_$(GOOS)) $(_SWIG_LDFLAGS_$(GOOS)) $(TARGDIR)/swig/$(SWIG_PREFIX)-%.so: _obj/$(SWIG_PREFIX)-%.so - @test -d $(QUOTED_GOROOT)/pkg && mkdir -p $(TARGDIR)/swig + @mkdir -p $(TARGDIR)/swig cp $< "$@" all: $(SWIG_SOS) diff --git a/src/cmd/5a/lex.c b/src/cmd/5a/lex.c index a04cda220..3978f1a6c 100644 --- a/src/cmd/5a/lex.c +++ b/src/cmd/5a/lex.c @@ -338,6 +338,8 @@ struct "NRMD", LTYPEI, ANRMD, */ + "SQRTF", LTYPEI, ASQRTF, + "SQRTD", LTYPEI, ASQRTD, "CMPF", LTYPEL, ACMPF, "CMPD", LTYPEL, ACMPD, "ADDF", LTYPEK, AADDF, diff --git a/src/cmd/5c/gc.h b/src/cmd/5c/gc.h index 9e9d1bd7d..549e0c88a 100644 --- a/src/cmd/5c/gc.h +++ b/src/cmd/5c/gc.h @@ -69,7 +69,7 @@ struct Adr Sym* sym; char type; - char reg; + uchar reg; char name; char etype; }; @@ -83,7 +83,7 @@ struct Prog Prog* link; int32 lineno; char as; - char reg; + uchar reg; uchar scond; }; #define P ((Prog*)0) diff --git a/src/cmd/5g/Makefile b/src/cmd/5g/Makefile index 6873fbc68..b47014a4e 100644 --- a/src/cmd/5g/Makefile +++ b/src/cmd/5g/Makefile @@ -15,16 +15,17 @@ HFILES=\ OFILES=\ ../5l/enam.$O\ - list.$O\ - galign.$O\ - gobj.$O\ - ggen.$O\ - gsubr.$O\ cgen.$O\ cgen64.$O\ cplx.$O\ - reg.$O\ + galign.$O\ + ggen.$O\ + gobj.$O\ + gsubr.$O\ + list.$O\ peep.$O\ + pgen.$O\ + reg.$O\ LIB=\ ../gc/gc.a\ diff --git a/src/cmd/5g/cgen.c b/src/cmd/5g/cgen.c index 4e5f7ebcd..76e2707fa 100644 --- a/src/cmd/5g/cgen.c +++ b/src/cmd/5g/cgen.c @@ -1190,7 +1190,8 @@ void sgen(Node *n, Node *res, int32 w) { Node dst, src, tmp, nend; - int32 c, q, odst, osrc; + int32 c, odst, osrc; + int dir, align, op; Prog *p, *ploop; if(debug['g']) { @@ -1200,17 +1201,39 @@ sgen(Node *n, Node *res, int32 w) } if(w == 0) return; - if(n->ullman >= UINF && res->ullman >= UINF) { - fatal("sgen UINF"); - } - if(w < 0) fatal("sgen copy %d", w); + if(n->ullman >= UINF && res->ullman >= UINF) + fatal("sgen UINF"); + if(n->type == T) + fatal("sgen: missing type"); + + // determine alignment. + // want to avoid unaligned access, so have to use + // smaller operations for less aligned types. + // for example moving [4]byte must use 4 MOVB not 1 MOVW. + align = n->type->align; + op = 0; + switch(align) { + default: + fatal("sgen: invalid alignment %d for %T", align, n->type); + case 1: + op = AMOVB; + break; + case 2: + op = AMOVH; + break; + case 4: + op = AMOVW; + break; + } + if(w%align) + fatal("sgen: unaligned size %d (align=%d) for %T", w, align, n->type); + c = w / align; // offset on the stack osrc = stkof(n); odst = stkof(res); - if(osrc != -1000 && odst != -1000 && (osrc == 1000 || odst == 1000)) { // osrc and odst both on stack, and at least one is in // an unknown position. Could generate code to test @@ -1221,12 +1244,15 @@ sgen(Node *n, Node *res, int32 w) sgen(&tmp, res, w); return; } - - if(osrc % 4 != 0 || odst %4 != 0) - fatal("sgen: non word(4) aligned offset src %d or dst %d", osrc, odst); + if(osrc%align != 0 || odst%align != 0) + fatal("sgen: unaligned offset src %d or dst %d (align %d)", osrc, odst, align); + // if we are copying forward on the stack and + // the src and dst overlap, then reverse direction + dir = align; + if(osrc < odst && odst < osrc+w) + dir = -dir; regalloc(&dst, types[tptr], res); - if(n->ullman >= res->ullman) { agen(n, &dst); // temporarily use dst regalloc(&src, types[tptr], N); @@ -1240,141 +1266,64 @@ sgen(Node *n, Node *res, int32 w) regalloc(&tmp, types[TUINT32], N); - c = w % 4; // bytes - q = w / 4; // quads - - // if we are copying forward on the stack and - // the src and dst overlap, then reverse direction - if(osrc < odst && odst < osrc+w) { - if(c != 0) - fatal("sgen: reverse character copy not implemented"); - if(q >= 4) { - regalloc(&nend, types[TUINT32], N); - // set up end marker to 4 bytes before source - p = gins(AMOVW, &src, &nend); - p->from.type = D_CONST; - p->from.offset = -4; - - // move src and dest to the end of block - p = gins(AMOVW, &src, &src); - p->from.type = D_CONST; - p->from.offset = (q-1)*4; - - p = gins(AMOVW, &dst, &dst); - p->from.type = D_CONST; - p->from.offset = (q-1)*4; - - p = gins(AMOVW, &src, &tmp); - p->from.type = D_OREG; - p->from.offset = -4; - p->scond |= C_PBIT; - ploop = p; + // set up end marker + memset(&nend, 0, sizeof nend); + if(c >= 4) { + regalloc(&nend, types[TUINT32], N); - p = gins(AMOVW, &tmp, &dst); - p->to.type = D_OREG; - p->to.offset = -4; - p->scond |= C_PBIT; - - p = gins(ACMP, &src, N); - raddr(&nend, p); + p = gins(AMOVW, &src, &nend); + p->from.type = D_CONST; + if(dir < 0) + p->from.offset = dir; + else + p->from.offset = w; + } - patch(gbranch(ABNE, T), ploop); + // move src and dest to the end of block if necessary + if(dir < 0) { + p = gins(AMOVW, &src, &src); + p->from.type = D_CONST; + p->from.offset = w + dir; - regfree(&nend); - } else { - // move src and dest to the end of block - p = gins(AMOVW, &src, &src); - p->from.type = D_CONST; - p->from.offset = (q-1)*4; - - p = gins(AMOVW, &dst, &dst); - p->from.type = D_CONST; - p->from.offset = (q-1)*4; - - while(q > 0) { - p = gins(AMOVW, &src, &tmp); - p->from.type = D_OREG; - p->from.offset = -4; - p->scond |= C_PBIT; - - p = gins(AMOVW, &tmp, &dst); - p->to.type = D_OREG; - p->to.offset = -4; - p->scond |= C_PBIT; - - q--; - } - } + p = gins(AMOVW, &dst, &dst); + p->from.type = D_CONST; + p->from.offset = w + dir; + } + + // move + if(c >= 4) { + p = gins(op, &src, &tmp); + p->from.type = D_OREG; + p->from.offset = dir; + p->scond |= C_PBIT; + ploop = p; + + p = gins(op, &tmp, &dst); + p->to.type = D_OREG; + p->to.offset = dir; + p->scond |= C_PBIT; + + p = gins(ACMP, &src, N); + raddr(&nend, p); + + patch(gbranch(ABNE, T), ploop); + regfree(&nend); } else { - // normal direction - if(q >= 4) { - regalloc(&nend, types[TUINT32], N); - p = gins(AMOVW, &src, &nend); - p->from.type = D_CONST; - p->from.offset = q*4; - - p = gins(AMOVW, &src, &tmp); + while(c-- > 0) { + p = gins(op, &src, &tmp); p->from.type = D_OREG; - p->from.offset = 4; + p->from.offset = dir; p->scond |= C_PBIT; ploop = p; - - p = gins(AMOVW, &tmp, &dst); + + p = gins(op, &tmp, &dst); p->to.type = D_OREG; - p->to.offset = 4; + p->to.offset = dir; p->scond |= C_PBIT; - - p = gins(ACMP, &src, N); - raddr(&nend, p); - - patch(gbranch(ABNE, T), ploop); - - regfree(&nend); - } else - while(q > 0) { - p = gins(AMOVW, &src, &tmp); - p->from.type = D_OREG; - p->from.offset = 4; - p->scond |= C_PBIT; - - p = gins(AMOVW, &tmp, &dst); - p->to.type = D_OREG; - p->to.offset = 4; - p->scond |= C_PBIT; - - q--; - } - - if (c != 0) { - // MOVW (src), tmp - p = gins(AMOVW, &src, &tmp); - p->from.type = D_OREG; - - // MOVW tmp<<((4-c)*8),src - gshift(AMOVW, &tmp, SHIFT_LL, ((4-c)*8), &src); - - // MOVW src>>((4-c)*8),src - gshift(AMOVW, &src, SHIFT_LR, ((4-c)*8), &src); - - // MOVW (dst), tmp - p = gins(AMOVW, &dst, &tmp); - p->from.type = D_OREG; - - // MOVW tmp>>(c*8),tmp - gshift(AMOVW, &tmp, SHIFT_LR, (c*8), &tmp); - - // MOVW tmp<<(c*8),tmp - gshift(AMOVW, &tmp, SHIFT_LL, c*8, &tmp); - - // ORR src, tmp - gins(AORR, &src, &tmp); - - // MOVW tmp, (dst) - p = gins(AMOVW, &tmp, &dst); - p->to.type = D_OREG; } } - regfree(&dst); + + regfree(&dst); regfree(&src); regfree(&tmp); } diff --git a/src/cmd/5g/gg.h b/src/cmd/5g/gg.h index 78e6833b2..ce4558e21 100644 --- a/src/cmd/5g/gg.h +++ b/src/cmd/5g/gg.h @@ -23,10 +23,11 @@ struct Addr char sval[NSNAME]; Sym* sym; + Node* node; int width; uchar type; char name; - char reg; + uchar reg; char pun; uchar etype; }; @@ -41,7 +42,7 @@ struct Prog Addr to; // dst address Prog* link; // next instruction in this func void* regp; // points to enclosing Reg struct - char reg; // doubles as width in DATA op + uchar reg; // doubles as width in DATA op uchar scond; }; diff --git a/src/cmd/5g/ggen.c b/src/cmd/5g/ggen.c index 7197709d4..0bc1b38fc 100644 --- a/src/cmd/5g/ggen.c +++ b/src/cmd/5g/ggen.c @@ -7,107 +7,9 @@ #include "gg.h" #include "opt.h" -static Prog *pret; - void -compile(Node *fn) +defframe(Prog *ptxt) { - Plist *pl; - Node nod1; - Prog *ptxt; - int32 lno; - Type *t; - Iter save; - - if(newproc == N) { - newproc = sysfunc("newproc"); - deferproc = sysfunc("deferproc"); - deferreturn = sysfunc("deferreturn"); - panicindex = sysfunc("panicindex"); - panicslice = sysfunc("panicslice"); - throwreturn = sysfunc("throwreturn"); - } - - if(fn->nbody == nil) - return; - - // set up domain for labels - clearlabels(); - - lno = setlineno(fn); - - curfn = fn; - dowidth(curfn->type); - - if(curfn->type->outnamed) { - // add clearing of the output parameters - t = structfirst(&save, getoutarg(curfn->type)); - while(t != T) { - if(t->nname != N) - curfn->nbody = concat(list1(nod(OAS, t->nname, N)), curfn->nbody); - t = structnext(&save); - } - } - - hasdefer = 0; - walk(curfn); - if(nerrors != 0 || isblank(curfn->nname)) - goto ret; - - allocparams(); - - continpc = P; - breakpc = P; - - pl = newplist(); - pl->name = curfn->nname; - - setlineno(curfn); - - nodconst(&nod1, types[TINT32], 0); - ptxt = gins(ATEXT, curfn->nname, &nod1); - afunclit(&ptxt->from); - - ginit(); - genlist(curfn->enter); - - pret = nil; - if(hasdefer || curfn->exit) { - Prog *p1; - - p1 = gjmp(nil); - pret = gjmp(nil); - patch(p1, pc); - } - - genlist(curfn->nbody); - gclean(); - checklabels(); - if(nerrors != 0) - goto ret; - - if(curfn->type->outtuple != 0) - ginscall(throwreturn, 0); - - if(pret) - patch(pret, pc); - ginit(); - if(hasdefer) - ginscall(deferreturn, 0); - if(curfn->exit) - genlist(curfn->exit); - gclean(); - if(nerrors != 0) - goto ret; - if(curfn->endlineno) - lineno = curfn->endlineno; - pc->as = ARET; // overwrite AEND - pc->lineno = lineno; - - if(!debug['N'] || debug['R'] || debug['P']) { - regopt(ptxt); - } - // fill in argument size ptxt->to.type = D_CONST2; ptxt->reg = 0; // flags @@ -118,12 +20,6 @@ compile(Node *fn) maxstksize = stksize; ptxt->to.offset = rnd(maxstksize+maxarg, widthptr); maxstksize = 0; - - if(debug['f']) - frame(0); - -ret: - lineno = lno; } /* @@ -197,7 +93,7 @@ ginscall(Node *f, int proc) nodconst(&con, types[TINT32], 0); p = gins(ACMP, &con, N); p->reg = 0; - patch(gbranch(ABNE, T), pret); + patch(gbranch(ABNE, T), retpc); } break; } @@ -401,7 +297,7 @@ cgen_ret(Node *n) { genlist(n->list); // copy out args if(hasdefer || curfn->exit) - gjmp(pret); + gjmp(retpc); else gins(ARET, N, N); } diff --git a/src/cmd/5g/gsubr.c b/src/cmd/5g/gsubr.c index 57cb56a64..caaa3e246 100644 --- a/src/cmd/5g/gsubr.c +++ b/src/cmd/5g/gsubr.c @@ -1159,6 +1159,7 @@ naddr(Node *n, Addr *a, int canemitcode) a->type = D_NONE; a->name = D_NONE; a->reg = NREG; + a->node = N; if(n == N) return; @@ -1247,6 +1248,8 @@ naddr(Node *n, Addr *a, int canemitcode) break; case PAUTO: a->name = D_AUTO; + if (n->sym) + a->node = n->orig; break; case PPARAM: case PPARAMOUT: diff --git a/src/cmd/5g/reg.c b/src/cmd/5g/reg.c index 1cbeb3e3d..5fba02c9e 100644 --- a/src/cmd/5g/reg.c +++ b/src/cmd/5g/reg.c @@ -32,6 +32,8 @@ #include "gg.h" #include "opt.h" +#define NREGVAR 24 +#define REGBITS ((uint32)0xffffff) #define P2R(p) (Reg*)(p->reg) void addsplits(void); @@ -128,6 +130,33 @@ setaddrs(Bits bit) } } +static char* regname[] = { + ".R0", + ".R1", + ".R2", + ".R3", + ".R4", + ".R5", + ".R6", + ".R7", + ".R8", + ".R9", + ".R10", + ".R11", + ".R12", + ".R13", + ".R14", + ".R15", + ".F0", + ".F1", + ".F2", + ".F3", + ".F4", + ".F5", + ".F6", + ".F7", +}; + void regopt(Prog *firstp) { @@ -136,7 +165,7 @@ regopt(Prog *firstp) int i, z, nr; uint32 vreg; Bits bit; - + if(first == 0) { fmtinstall('Q', Qconv); } @@ -164,7 +193,17 @@ regopt(Prog *firstp) r1 = R; firstr = R; lastr = R; - nvar = 0; + + /* + * control flow is more complicated in generated go code + * than in generated c code. define pseudo-variables for + * registers, so we have complete register usage information. + */ + nvar = NREGVAR; + memset(var, 0, NREGVAR*sizeof var[0]); + for(i=0; i<NREGVAR; i++) + var[i].sym = lookup(regname[i]); + regbits = RtoB(REGSP)|RtoB(REGLINK)|RtoB(REGPC); for(z=0; z<BITS; z++) { externs.b[z] = 0; @@ -223,6 +262,16 @@ regopt(Prog *firstp) bit = mkvar(r, &p->from); for(z=0; z<BITS; z++) r->use1.b[z] |= bit.b[z]; + + /* + * middle always read when present + */ + if(p->reg != NREG) { + if(p->from.type != D_FREG) + r->use1.b[0] |= RtoB(p->reg); + else + r->use1.b[0] |= FtoB(p->reg); + } /* * right side depends on opcode @@ -233,6 +282,67 @@ regopt(Prog *firstp) default: yyerror("reg: unknown op: %A", p->as); break; + + /* + * right side read + */ + case ATST: + case ATEQ: + case ACMP: + case ACMN: + case ACMPD: + case ACMPF: + rightread: + for(z=0; z<BITS; z++) + r->use2.b[z] |= bit.b[z]; + break; + + /* + * right side read or read+write, depending on middle + * ADD x, z => z += x + * ADD x, y, z => z = x + y + */ + case AADD: + case AAND: + case AEOR: + case ASUB: + case ARSB: + case AADC: + case ASBC: + case ARSC: + case AORR: + case ABIC: + case ASLL: + case ASRL: + case ASRA: + case AMUL: + case AMULU: + case ADIV: + case AMOD: + case AMODU: + case ADIVU: + if(p->reg != NREG) + goto rightread; + // fall through + + /* + * right side read+write + */ + case AADDF: + case AADDD: + case ASUBF: + case ASUBD: + case AMULF: + case AMULD: + case ADIVF: + case ADIVD: + case AMULAL: + case AMULALU: + for(z=0; z<BITS; z++) { + r->use2.b[z] |= bit.b[z]; + r->set.b[z] |= bit.b[z]; + } + break; /* * right side write @@ -240,11 +350,22 @@ regopt(Prog *firstp) case ANOP: case AMOVB: case AMOVBU: + case AMOVD: + case AMOVDF: + case AMOVDW: + case AMOVF: + case AMOVFW: case AMOVH: case AMOVHU: case AMOVW: - case AMOVF: - case AMOVD: + case AMOVWD: + case AMOVWF: + case AMVN: + case AMULL: + case AMULLU: + if((p->scond & C_SCOND) != C_SCOND_NONE) + for(z=0; z<BITS; z++) + r->use2.b[z] |= bit.b[z]; for(z=0; z<BITS; z++) r->set.b[z] |= bit.b[z]; break; @@ -397,6 +518,24 @@ loop2: } /* + * pass 4.5 + * move register pseudo-variables into regu. + */ + for(r = firstr; r != R; r = r->link) { + r->regu = (r->refbehind.b[0] | r->set.b[0]) & REGBITS; + + r->set.b[0] &= ~REGBITS; + r->use1.b[0] &= ~REGBITS; + r->use2.b[0] &= ~REGBITS; + r->refbehind.b[0] &= ~REGBITS; + r->refahead.b[0] &= ~REGBITS; + r->calbehind.b[0] &= ~REGBITS; + r->calahead.b[0] &= ~REGBITS; + r->regdiff.b[0] &= ~REGBITS; + r->act.b[0] &= ~REGBITS; + } + + /* * pass 5 * isolate regions * calculate costs (paint1) @@ -606,6 +745,7 @@ addmove(Reg *r, int bn, int rn, int f) a = &p1->to; a->sym = v->sym; a->name = v->name; + a->node = v->node; a->offset = v->offset; a->etype = v->etype; a->type = D_OREG; @@ -715,21 +855,40 @@ mkvar(Reg *r, Adr *a) goto onereg; case D_REGREG: + bit = zbits; if(a->offset != NREG) - r->regu |= RtoB(a->offset); - goto onereg; + bit.b[0] |= RtoB(a->offset); + if(a->reg != NREG) + bit.b[0] |= RtoB(a->reg); + return bit; case D_REG: case D_SHIFT: - case D_OREG: onereg: - if(a->reg != NREG) - r->regu |= RtoB(a->reg); + if(a->reg != NREG) { + bit = zbits; + bit.b[0] = RtoB(a->reg); + return bit; + } + break; + + case D_OREG: + if(a->reg != NREG) { + if(a == &r->prog->from) + r->use1.b[0] |= RtoB(a->reg); + else + r->use2.b[0] |= RtoB(a->reg); + if(r->prog->scond & (C_PBIT|C_WBIT)) + r->set.b[0] |= RtoB(a->reg); + } break; case D_FREG: - if(a->reg != NREG) - r->regu |= FtoB(a->reg); + if(a->reg != NREG) { + bit = zbits; + bit.b[0] = FtoB(a->reg); + return bit; + } break; } @@ -795,7 +954,8 @@ mkvar(Reg *r, Adr *a) v->etype = et; v->width = w; v->addr = flag; // funny punning - + v->node = a->node; + if(debug['R']) print("bit=%2d et=%E pun=%d %D\n", i, et, flag, a); diff --git a/src/cmd/5l/5.out.h b/src/cmd/5l/5.out.h index 002b46d45..cf86ae48b 100644 --- a/src/cmd/5l/5.out.h +++ b/src/cmd/5l/5.out.h @@ -53,8 +53,6 @@ #define REGLINK 14 #define REGPC 15 -#define REGTMPT 7 /* used by the loader for thumb code */ - #define NFREG 8 #define FREGRET 0 #define FREGEXT 7 @@ -126,6 +124,8 @@ enum as AMULD, ADIVF, ADIVD, + ASQRTF, + ASQRTD, ASRL, ASRA, diff --git a/src/cmd/5l/Makefile b/src/cmd/5l/Makefile index c11ebe990..9f4a192aa 100644 --- a/src/cmd/5l/Makefile +++ b/src/cmd/5l/Makefile @@ -22,7 +22,6 @@ OFILES=\ optab.$O\ pass.$O\ prof.$O\ - thumb.$O\ softfloat.$O\ span.$O\ symtab.$O\ diff --git a/src/cmd/5l/asm.c b/src/cmd/5l/asm.c index e2583e7c3..4afed2b80 100644 --- a/src/cmd/5l/asm.c +++ b/src/cmd/5l/asm.c @@ -34,8 +34,6 @@ #include "../ld/lib.h" #include "../ld/elf.h" -int32 OFFSET; - static Prog *PP; char linuxdynld[] = "/lib/ld-linux.so.2"; @@ -73,6 +71,8 @@ enum { ElfStrGosymcounts, ElfStrGosymtab, ElfStrGopclntab, + ElfStrSymtab, + ElfStrStrtab, ElfStrShstrtab, ElfStrRelPlt, ElfStrPlt, @@ -87,6 +87,9 @@ needlib(char *name) char *p; Sym *s; + if(*name == '\0') + return 0; + /* reuse hash code in symbol table */ p = smprint(".dynlib.%s", name); s = lookup(p, 0); @@ -163,6 +166,8 @@ doelf(void) elfstr[ElfStrGosymcounts] = addstring(shstrtab, ".gosymcounts"); elfstr[ElfStrGosymtab] = addstring(shstrtab, ".gosymtab"); elfstr[ElfStrGopclntab] = addstring(shstrtab, ".gopclntab"); + elfstr[ElfStrSymtab] = addstring(shstrtab, ".symtab"); + elfstr[ElfStrStrtab] = addstring(shstrtab, ".strtab"); } elfstr[ElfStrShstrtab] = addstring(shstrtab, ".shstrtab"); @@ -288,15 +293,12 @@ asmb(void) { int32 t; int a, dynsym; - uint32 va, fo, w, startva; - int strtabsize; + uint32 fo, symo, startva; ElfEhdr *eh; ElfPhdr *ph, *pph; ElfShdr *sh; Section *sect; - strtabsize = 0; - if(debug['v']) Bprint(&bso, "%5.2f asmb\n", cputime()); Bflush(&bso); @@ -322,15 +324,30 @@ asmb(void) seek(cout, sect->vaddr - segtext.vaddr + segtext.fileoff, 0); datblk(sect->vaddr, sect->len); + if(iself) { + /* index of elf text section; needed by asmelfsym, double-checked below */ + /* !debug['d'] causes extra sections before the .text section */ + elftextsh = 1; + if(!debug['d']) { + elftextsh += 10; + if(elfverneed) + elftextsh += 2; + } + } + /* output symbol table */ symsize = 0; lcsize = 0; + symo = 0; if(!debug['s']) { // TODO: rationalize if(debug['v']) Bprint(&bso, "%5.2f sym\n", cputime()); Bflush(&bso); switch(HEADTYPE) { + default: + if(iself) + goto ElfSym; case Hnoheader: case Hrisc: case Hixp1200: @@ -338,29 +355,37 @@ asmb(void) debug['s'] = 1; break; case Hplan9x32: - OFFSET = HEADR+textsize+segdata.filelen; - seek(cout, OFFSET, 0); + symo = HEADR+segtext.len+segdata.filelen; break; case Hnetbsd: - OFFSET += rnd(segdata.filelen, 4096); - seek(cout, OFFSET, 0); + symo = rnd(segdata.filelen, 4096); break; - case Hlinux: - OFFSET += segdata.filelen; - seek(cout, rnd(OFFSET, INITRND), 0); + ElfSym: + symo = rnd(HEADR+segtext.filelen, INITRND)+segdata.filelen; + symo = rnd(symo, INITRND); break; } - if(!debug['s']) - asmthumbmap(); + seek(cout, symo, 0); + if(iself) { + if(debug['v']) + Bprint(&bso, "%5.2f elfsym\n", cputime()); + asmelfsym(); + cflush(); + ewrite(cout, elfstrdat, elfstrsize); + + // if(debug['v']) + // Bprint(&bso, "%5.2f dwarf\n", cputime()); + // dwarfemitdebugsections(); + } cflush(); + } cursym = nil; if(debug['v']) Bprint(&bso, "%5.2f header\n", cputime()); Bflush(&bso); - OFFSET = 0; - seek(cout, OFFSET, 0); + seek(cout, 0L, 0); switch(HEADTYPE) { case Hnoheader: /* no header */ break; @@ -426,9 +451,7 @@ asmb(void) /* elf arm */ eh = getElfEhdr(); fo = HEADR; - va = INITTEXT; startva = INITTEXT - fo; /* va of byte 0 of file */ - w = textsize; /* This null SHdr must appear before all others */ sh = newElfShdr(elfstr[ElfStrEmpty]); @@ -541,6 +564,8 @@ asmb(void) ph->flags = PF_W+PF_R; ph->align = 4; + if(elftextsh != eh->shnum) + diag("elftextsh = %d, want %d", elftextsh, eh->shnum); for(sect=segtext.sect; sect!=nil; sect=sect->next) elfshbits(sect); for(sect=segdata.sect; sect!=nil; sect=sect->next) @@ -558,6 +583,22 @@ asmb(void) sh->flags = SHF_ALLOC; sh->addralign = 1; shsym(sh, lookup("pclntab", 0)); + + sh = newElfShdr(elfstr[ElfStrSymtab]); + sh->type = SHT_SYMTAB; + sh->off = symo; + sh->size = symsize; + sh->addralign = 4; + sh->entsize = 16; + sh->link = eh->shnum; // link to strtab + + sh = newElfShdr(elfstr[ElfStrStrtab]); + sh->type = SHT_STRTAB; + sh->off = symo+symsize; + sh->size = elfstrsize; + sh->addralign = 1; + + // dwarfaddelfheaders(); } sh = newElfShstrtab(elfstr[ElfStrShstrtab]); @@ -686,59 +727,6 @@ nopstat(char *f, Count *c) (double)(c->outof - c->count)/c->outof); } -static void -outt(int32 f, int32 l) -{ - if(debug['L']) - Bprint(&bso, "tmap: %ux-%ux\n", f, l); - lput(f); - lput(l); -} - -void -asmthumbmap(void) -{ - int32 pc, lastt; - Prog *p; - - if(!seenthumb) - return; - pc = 0; - lastt = -1; - for(cursym = textp; cursym != nil; cursym = cursym->next) { - p = cursym->text; - pc = p->pc - INITTEXT; - setarch(p); - if(thumb){ - if(p->from.sym->foreign){ // 8 bytes of ARM first - if(lastt >= 0){ - outt(lastt, pc-1); - lastt = -1; - } - pc += 8; - } - if(lastt < 0) - lastt = pc; - } - else{ - if(p->from.sym->foreign){ // 4 bytes of THUMB first - if(lastt < 0) - lastt = pc; - pc += 4; - } - if(lastt >= 0){ - outt(lastt, pc-1); - lastt = -1; - } - } - if(cursym->next == nil) - for(; p != P; p = p->link) - pc = p->pc = INITTEXT; - } - if(lastt >= 0) - outt(lastt, pc+1); -} - void asmout(Prog *p, Optab *o, int32 *out) { @@ -762,7 +750,7 @@ if(debug['P']) print("%ux: %P type %d\n", (uint32)(p->pc), p, o->type); break; case 0: /* pseudo ops */ -if(debug['G']) print("%ux: %s: arm %d %d %d\n", (uint32)(p->pc), p->from.sym->name, p->from.sym->thumb, p->from.sym->foreign, p->from.sym->fnptr); +if(debug['G']) print("%ux: %s: arm %d\n", (uint32)(p->pc), p->from.sym->name, p->from.sym->fnptr); break; case 1: /* op R,[R],R */ @@ -826,10 +814,6 @@ if(debug['G']) print("%ux: %s: arm %d %d %d\n", (uint32)(p->pc), p->from.sym->na v = -8; if(p->cond != P) v = (p->cond->pc - pc) - 8; -#ifdef CALLEEBX - if(p->as == ABL) - v += fninc(p->to.sym); -#endif o1 = opbra(p->as, p->scond); o1 |= (v >> 2) & 0xffffff; break; @@ -990,40 +974,6 @@ if(debug['G']) print("%ux: %s: arm %d %d %d\n", (uint32)(p->pc), p->from.sym->na o1 |= 1<<22; break; - case 22: /* movb/movh/movhu O(R),R -> lr,shl,shr */ - aclass(&p->from); - r = p->from.reg; - if(r == NREG) - r = o->param; - o1 = olr(instoffset, r, p->to.reg, p->scond); - - o2 = oprrr(ASLL, p->scond); - o3 = oprrr(ASRA, p->scond); - r = p->to.reg; - if(p->as == AMOVB) { - o2 |= (24<<7)|(r)|(r<<12); - o3 |= (24<<7)|(r)|(r<<12); - } else { - o2 |= (16<<7)|(r)|(r<<12); - if(p->as == AMOVHU) - o3 = oprrr(ASRL, p->scond); - o3 |= (16<<7)|(r)|(r<<12); - } - break; - - case 23: /* movh/movhu R,O(R) -> sb,sb */ - aclass(&p->to); - r = p->to.reg; - if(r == NREG) - r = o->param; - o1 = osr(AMOVH, p->from.reg, instoffset, r, p->scond); - - o2 = oprrr(ASRL, p->scond); - o2 |= (8<<7)|(p->from.reg)|(REGTMP<<12); - - o3 = osr(AMOVH, REGTMP, instoffset+1, r, p->scond); - break; - case 30: /* mov/movb/movbu R,L(R) */ o1 = omvl(p, &p->to, REGTMP); if(!o1) @@ -1037,7 +987,6 @@ if(debug['G']) print("%ux: %s: arm %d %d %d\n", (uint32)(p->pc), p->from.sym->na break; case 31: /* mov/movbu L(R),R -> lr[b] */ - case 32: /* movh/movb L(R),R -> lr[b] */ o1 = omvl(p, &p->from, REGTMP); if(!o1) break; @@ -1047,53 +996,6 @@ if(debug['G']) print("%ux: %s: arm %d %d %d\n", (uint32)(p->pc), p->from.sym->na o2 = olrr(REGTMP,r, p->to.reg, p->scond); if(p->as == AMOVBU || p->as == AMOVB) o2 |= 1<<22; - if(o->type == 31) - break; - - o3 = oprrr(ASLL, p->scond); - - if(p->as == AMOVBU || p->as == AMOVHU) - o4 = oprrr(ASRL, p->scond); - else - o4 = oprrr(ASRA, p->scond); - - r = p->to.reg; - o3 |= (r)|(r<<12); - o4 |= (r)|(r<<12); - if(p->as == AMOVB || p->as == AMOVBU) { - o3 |= (24<<7); - o4 |= (24<<7); - } else { - o3 |= (16<<7); - o4 |= (16<<7); - } - break; - - case 33: /* movh/movhu R,L(R) -> sb, sb */ - o1 = omvl(p, &p->to, REGTMP); - if(!o1) - break; - r = p->to.reg; - if(r == NREG) - r = o->param; - o2 = osrr(p->from.reg, REGTMP, r, p->scond); - o2 |= (1<<22) ; - - o3 = oprrr(ASRL, p->scond); - o3 |= (8<<7)|(p->from.reg)|(p->from.reg<<12); - o3 |= (1<<6); /* ROR 8 */ - - o4 = oprrr(AADD, p->scond); - o4 |= (REGTMP << 12) | (REGTMP << 16); - o4 |= immrot(1); - - o5 = osrr(p->from.reg, REGTMP,r,p->scond); - o5 |= (1<<22); - - o6 = oprrr(ASRL, p->scond); - o6 |= (24<<7)|(p->from.reg)|(p->from.reg<<12); - o6 |= (1<<6); /* ROL 8 */ - break; case 34: /* mov $lacon,R */ @@ -1226,7 +1128,7 @@ if(debug['G']) print("%ux: %s: arm %d %d %d\n", (uint32)(p->pc), p->from.sym->na r = p->reg; if(r == NREG) { r = rt; - if(p->as == AMOVF || p->as == AMOVD) + if(p->as == AMOVF || p->as == AMOVD || p->as == ASQRTF || p->as == ASQRTD) r = 0; } o1 |= rf | (r<<16) | (rt<<12); @@ -1304,54 +1206,12 @@ if(debug['G']) print("%ux: %s: arm %d %d %d\n", (uint32)(p->pc), p->from.sym->na break; case 65: /* mov/movbu addr,R */ - case 66: /* movh/movhu/movb addr,R */ o1 = omvl(p, &p->from, REGTMP); if(!o1) break; o2 = olr(0, REGTMP, p->to.reg, p->scond); if(p->as == AMOVBU || p->as == AMOVB) o2 |= 1<<22; - if(o->type == 65) - break; - - o3 = oprrr(ASLL, p->scond); - - if(p->as == AMOVBU || p->as == AMOVHU) - o4 = oprrr(ASRL, p->scond); - else - o4 = oprrr(ASRA, p->scond); - - r = p->to.reg; - o3 |= (r)|(r<<12); - o4 |= (r)|(r<<12); - if(p->as == AMOVB || p->as == AMOVBU) { - o3 |= (24<<7); - o4 |= (24<<7); - } else { - o3 |= (16<<7); - o4 |= (16<<7); - } - break; - - case 67: /* movh/movhu R,addr -> sb, sb */ - o1 = omvl(p, &p->to, REGTMP); - if(!o1) - break; - o2 = osr(p->as, p->from.reg, 0, REGTMP, p->scond); - - o3 = oprrr(ASRL, p->scond); - o3 |= (8<<7)|(p->from.reg)|(p->from.reg<<12); - o3 |= (1<<6); /* ROR 8 */ - - o4 = oprrr(AADD, p->scond); - o4 |= (REGTMP << 12) | (REGTMP << 16); - o4 |= immrot(1); - - o5 = osr(p->as, p->from.reg, 0, REGTMP, p->scond); - - o6 = oprrr(ASRL, p->scond); - o6 |= (24<<7)|(p->from.reg)|(p->from.reg<<12); - o6 |= (1<<6); /* ROL 8 */ break; case 68: /* floating point store -> ADDR */ @@ -1410,19 +1270,7 @@ if(debug['G']) print("%ux: %s: arm %d %d %d\n", (uint32)(p->pc), p->from.sym->na o2 ^= (1<<6); break; case 74: /* bx $I */ -#ifdef CALLEEBX - diag("bx $i case (arm)"); -#endif - if(!seenthumb) - diag("ABX $I and seenthumb==0"); - v = p->cond->pc; - if(p->to.sym->thumb) - v |= 1; // T bit - o1 = olr(8, REGPC, REGTMP, p->scond&C_SCOND); // mov 8(PC), Rtmp - o2 = oprrr(AADD, p->scond) | immrot(8) | (REGPC<<16) | (REGLINK<<12); // add 8,PC, LR - o3 = ((p->scond&C_SCOND)<<28) | (0x12fff<<8) | (1<<4) | REGTMP; // bx Rtmp - o4 = opbra(AB, 14); // B over o6 - o5 = v; + diag("ABX $I"); break; case 75: /* bx O(R) */ aclass(&p->to); @@ -1441,14 +1289,7 @@ if(debug['G']) print("%ux: %s: arm %d %d %d\n", (uint32)(p->pc), p->from.sym->na o3 = ((p->scond&C_SCOND)<<28) | (0x12fff<<8) | (1<<4) | REGTMP; // BX Rtmp break; case 76: /* bx O(R) when returning from fn*/ - if(!seenthumb) - diag("ABXRET and seenthumb==0"); - aclass(&p->to); -// print("ARM BXRET %d(R%d)\n", instoffset, p->to.reg); - if(instoffset != 0) - diag("non-zero offset in ABXRET"); - // o1 = olr(instoffset, p->to.reg, REGTMP, p->scond); // mov O(R), Rtmp - o1 = ((p->scond&C_SCOND)<<28) | (0x12fff<<8) | (1<<4) | p->to.reg; // BX R + diag("ABXRET"); break; case 77: /* ldrex oreg,reg */ aclass(&p->from); @@ -1572,6 +1413,22 @@ if(debug['G']) print("%ux: %s: arm %d %d %d\n", (uint32)(p->pc), p->from.sym->na o1 |= p->to.reg << 12; o1 |= (p->scond & C_SCOND) << 28; break; + case 93: /* movb/movh/movhu addr,R -> ldrsb/ldrsh/ldrh */ + o1 = omvl(p, &p->from, REGTMP); + if(!o1) + break; + o2 = olhr(0, REGTMP, p->to.reg, p->scond); + if(p->as == AMOVB) + o2 ^= (1<<5)|(1<<6); + else if(p->as == AMOVH) + o2 ^= (1<<6); + break; + case 94: /* movh/movhu R,addr -> strh */ + o1 = omvl(p, &p->to, REGTMP); + if(!o1) + break; + o2 = oshr(p->from.reg, 0, REGTMP, p->scond); + break; } out[0] = o1; @@ -1686,6 +1543,8 @@ oprrr(int a, int sc) case AMULF: return o | (0xe<<24) | (0x2<<20) | (0xa<<8) | (0<<4); case ADIVD: return o | (0xe<<24) | (0x8<<20) | (0xb<<8) | (0<<4); case ADIVF: return o | (0xe<<24) | (0x8<<20) | (0xa<<8) | (0<<4); + case ASQRTD: return o | (0xe<<24) | (0xb<<20) | (1<<16) | (0xb<<8) | (0xc<<4); + case ASQRTF: return o | (0xe<<24) | (0xb<<20) | (1<<16) | (0xa<<8) | (0xc<<4); case ACMPD: return o | (0xe<<24) | (0xb<<20) | (4<<16) | (0xb<<8) | (0xc<<4); case ACMPF: return o | (0xe<<24) | (0xb<<20) | (4<<16) | (0xa<<8) | (0xc<<4); @@ -2031,3 +1890,9 @@ genasmsym(void (*put)(Sym*, char*, int, vlong, vlong, int, Sym*)) Bprint(&bso, "symsize = %ud\n", symsize); Bflush(&bso); } + +void +setpersrc(Sym *s) +{ + USED(s); +} diff --git a/src/cmd/5l/l.h b/src/cmd/5l/l.h index f3c9d839d..182f3e738 100644 --- a/src/cmd/5l/l.h +++ b/src/cmd/5l/l.h @@ -144,8 +144,6 @@ struct Sym int32 sig; int32 size; uchar special; - uchar thumb; // thumb code - uchar foreign; // called by arm if thumb, by thumb if arm uchar fnptr; // used as fn ptr Sym* hash; // in hash table Sym* allsym; // in all symbol list @@ -208,7 +206,6 @@ enum LFROM = 1<<0, LTO = 1<<1, LPOOL = 1<<2, - V4 = 1<<3, /* arm v4 arch */ C_NONE = 0, C_REG, @@ -221,21 +218,16 @@ enum C_RCON, /* 0xff rotated */ C_NCON, /* ~RCON */ C_SCON, /* 0xffff */ - C_BCON, /* thumb */ C_LCON, C_ZFCON, C_SFCON, C_LFCON, - C_GCON, /* thumb */ C_RACON, - C_SACON, /* thumb */ C_LACON, - C_GACON, /* thumb */ C_SBRA, C_LBRA, - C_GBRA, /* thumb */ C_HAUTO, /* halfword insn offset (-0xff to 0xff) */ C_FAUTO, /* float insn offset (0 to 0x3fc, word aligned) */ @@ -250,12 +242,10 @@ enum C_ROREG, C_SROREG, /* both S and R */ C_LOREG, - C_GOREG, /* thumb */ C_PC, C_SP, C_HREG, - C_OFFPC, /* thumb */ C_ADDR, /* reference to relocatable address */ @@ -287,9 +277,6 @@ EXTERN union #define cbuf u.obuf #define xbuf u.ibuf -#define setarch(p) if((p)->as==ATEXT) thumb=(p)->reg&ALLTHUMBS -#define setthumb(p) if((p)->as==ATEXT) seenthumb|=(p)->reg&ALLTHUMBS - #ifndef COFFCVT EXTERN int32 HEADR; /* length of header */ @@ -319,7 +306,6 @@ EXTERN int nerrors; EXTERN int32 instoffset; EXTERN Opcross opcross[8]; EXTERN Oprang oprange[ALAST]; -EXTERN Oprang thumboprange[ALAST]; EXTERN char* outfile; EXTERN int32 pc; EXTERN uchar repop[ALAST]; @@ -333,14 +319,10 @@ EXTERN int version; EXTERN char xcmp[C_GOK+1][C_GOK+1]; EXTERN Prog zprg; EXTERN int dtype; -EXTERN int armv4; -EXTERN int thumb; -EXTERN int seenthumb; EXTERN int armsize; extern char* anames[]; extern Optab optab[]; -extern Optab thumboptab[]; void addpool(Prog*, Adr*); EXTERN Prog* blitrl; @@ -368,17 +350,13 @@ int Oconv(Fmt*); int Pconv(Fmt*); int Sconv(Fmt*); int aclass(Adr*); -int thumbaclass(Adr*, Prog*); void addhist(int32, int); Prog* appendp(Prog*); void asmb(void); -void asmthumbmap(void); void asmout(Prog*, Optab*, int32*); -void thumbasmout(Prog*, Optab*); int32 atolwhex(char*); Prog* brloop(Prog*); void buildop(void); -void thumbbuildop(void); void buildrep(int, int); void cflush(void); int chipzero(Ieee*); @@ -442,9 +420,6 @@ int32 immaddr(int32); int32 opbra(int, int); int brextra(Prog*); int isbranch(Prog*); -int fnpinc(Sym *); -int fninc(Sym *); -void thumbcount(void); void fnptrs(void); void doelf(void); diff --git a/src/cmd/5l/list.c b/src/cmd/5l/list.c index 2ae25d491..fa838215b 100644 --- a/src/cmd/5l/list.c +++ b/src/cmd/5l/list.c @@ -412,7 +412,6 @@ static char* cnames[] = { [C_ADDR] = "C_ADDR", - [C_BCON] = "C_BCON", [C_FAUTO] = "C_FAUTO", [C_ZFCON] = "C_SFCON", [C_SFCON] = "C_SFCON", @@ -420,11 +419,7 @@ cnames[] = [C_FCR] = "C_FCR", [C_FOREG] = "C_FOREG", [C_FREG] = "C_FREG", - [C_GACON] = "C_GACON", - [C_GBRA] = "C_GBRA", - [C_GCON] = "C_GCON", [C_GOK] = "C_GOK", - [C_GOREG] = "C_GOREG", [C_HAUTO] = "C_HAUTO", [C_HFAUTO] = "C_HFAUTO", [C_HFOREG] = "C_HFOREG", @@ -437,7 +432,6 @@ cnames[] = [C_LOREG] = "C_LOREG", [C_NCON] = "C_NCON", [C_NONE] = "C_NONE", - [C_OFFPC] = "C_OFFPC", [C_PC] = "C_PC", [C_PSR] = "C_PSR", [C_RACON] = "C_RACON", @@ -445,7 +439,6 @@ cnames[] = [C_REG] = "C_REG", [C_REGREG] = "C_REGREG", [C_ROREG] = "C_ROREG", - [C_SACON] = "C_SACON", [C_SAUTO] = "C_SAUTO", [C_SBRA] = "C_SBRA", [C_SCON] = "C_SCON", diff --git a/src/cmd/5l/noop.c b/src/cmd/5l/noop.c index bdcc3cad8..e7c2db5f2 100644 --- a/src/cmd/5l/noop.c +++ b/src/cmd/5l/noop.c @@ -47,74 +47,11 @@ static Sym* sym_modu; static void setdiv(int); -static Prog * -movrr(Prog *q, int rs, int rd, Prog *p) -{ - if(q == nil) - q = prg(); - q->as = AMOVW; - q->line = p->line; - q->from.type = D_REG; - q->from.reg = rs; - q->to.type = D_REG; - q->to.reg = rd; - q->link = p->link; - return q; -} - -static Prog * -fnret(Prog *q, int rs, int foreign, Prog *p) -{ - q = movrr(q, rs, REGPC, p); - if(foreign){ // BX rs - q->as = ABXRET; - q->from.type = D_NONE; - q->from.reg = NREG; - q->to.reg = rs; - } - return q; -} - -static Prog * -aword(int32 w, Prog *p) -{ - Prog *q; - - q = prg(); - q->as = AWORD; - q->line = p->line; - q->from.type = D_NONE; - q->reg = NREG; - q->to.type = D_CONST; - q->to.offset = w; - q->link = p->link; - p->link = q; - return q; -} - -static Prog * -adword(int32 w1, int32 w2, Prog *p) -{ - Prog *q; - - q = prg(); - q->as = ADWORD; - q->line = p->line; - q->from.type = D_CONST; - q->from.offset = w1; - q->reg = NREG; - q->to.type = D_CONST; - q->to.offset = w2; - q->link = p->link; - p->link = q; - return q; -} - void noops(void) { - Prog *p, *q, *q1, *q2; - int o, foreign; + Prog *p, *q, *q1; + int o; Prog *pmorestack; Sym *symmorestack; @@ -140,8 +77,6 @@ noops(void) q = P; for(cursym = textp; cursym != nil; cursym = cursym->next) { for(p = cursym->text; p != P; p = p->link) { - setarch(p); - switch(p->as) { case ATEXT: p->mark |= LEAF; @@ -206,7 +141,6 @@ noops(void) for(cursym = textp; cursym != nil; cursym = cursym->next) { for(p = cursym->text; p != P; p = p->link) { - setarch(p); o = p->as; switch(o) { case ATEXT: @@ -224,54 +158,12 @@ noops(void) Bflush(&bso); cursym->text->mark |= LEAF; } -#ifdef CALLEEBX - if(p->from.sym->foreign){ - if(thumb) - // don't allow literal pool to separate these - p = adword(0xe28f7001, 0xe12fff17, p); // arm add 1, pc, r7 and bx r7 - // p = aword(0xe12fff17, aword(0xe28f7001, p)); // arm add 1, pc, r7 and bx r7 - else - p = aword(0x4778, p); // thumb bx pc and 2 bytes padding - } -#endif if(cursym->text->mark & LEAF) { cursym->leaf = 1; if(!autosize) break; } - if(thumb){ - if(!(p->reg & NOSPLIT)) - diag("stack splitting not supported in thumb"); - if(!(cursym->text->mark & LEAF)){ - q = movrr(nil, REGLINK, REGTMPT-1, p); - p->link = q; - q1 = prg(); - q1->as = AMOVW; - q1->line = p->line; - q1->from.type = D_REG; - q1->from.reg = REGTMPT-1; - q1->to.type = D_OREG; - q1->to.name = D_NONE; - q1->to.reg = REGSP; - q1->to.offset = 0; - q1->link = q->link; - q->link = q1; - } - if(autosize){ - q2 = prg(); - q2->as = ASUB; - q2->line = p->line; - q2->from.type = D_CONST; - q2->from.offset = autosize; - q2->to.type = D_REG; - q2->to.reg = REGSP; - q2->link = p->link; - p->link = q2; - } - break; - } - if(p->reg & NOSPLIT) { q1 = prg(); q1->as = AMOVW; @@ -432,16 +324,9 @@ noops(void) case ARET: nocache(p); - foreign = seenthumb && (cursym->foreign || cursym->fnptr); -// print("%s %d %d\n", cursym->name, cursym->foreign, cursym->fnptr); if(cursym->text->mark & LEAF) { if(!autosize) { - if(thumb){ - p = fnret(p, REGLINK, foreign, p); - break; - } -// if(foreign) print("ABXRET 1 %s\n", cursym->name); - p->as = foreign ? ABXRET : AB; + p->as = AB; p->from = zprg.from; p->to.type = D_OREG; p->to.offset = 0; @@ -449,95 +334,16 @@ noops(void) break; } } - if(thumb){ - diag("thumb not maintained"); - errorexit(); - if(cursym->text->mark & LEAF){ - if(autosize){ - p->as = AADD; - p->from.type = D_CONST; - p->from.offset = autosize; - p->to.type = D_REG; - p->to.reg = REGSP; - q = nil; - } - else - q = p; - q = fnret(q, REGLINK, foreign, p); - if(q != p) - p->link = q; - } - else{ - p->as = AMOVW; - p->from.type = D_OREG; - p->from.name = D_NONE; - p->from.reg = REGSP; - p->from.offset = 0; - p->to.type = D_REG; - p->to.reg = REGTMPT-1; - if(autosize){ - q = prg(); - q->as = AADD; - q->from.type = D_CONST; - q->from.offset = autosize; - q->to.type = D_REG; - q->to.reg = REGSP; - q->link = p->link; - p->link = q; - } - else - q = p; - q1 = fnret(nil, REGTMPT-1, foreign, p); - q1->link = q->link; - q->link = q1; - } - break; - } - if(foreign) { - diag("foreign not maintained"); - errorexit(); -// if(foreign) print("ABXRET 3 %s\n", cursym->name); -#define R 1 - p->as = AMOVW; - p->from.type = D_OREG; - p->from.name = D_NONE; - p->from.reg = REGSP; - p->from.offset = 0; - p->to.type = D_REG; - p->to.reg = R; - q = prg(); - q->as = AADD; - q->scond = p->scond; - q->line = p->line; - q->from.type = D_CONST; - q->from.offset = autosize; - q->to.type = D_REG; - q->to.reg = REGSP; - q->link = p->link; - p->link = q; - q1 = prg(); - q1->as = ABXRET; - q1->scond = p->scond; - q1->line = p->line; - q1->to.type = D_OREG; - q1->to.offset = 0; - q1->to.reg = R; - q1->link = q->link; - q->link = q1; -#undef R - } - else { - p->as = AMOVW; - p->scond |= C_PBIT; - p->from.type = D_OREG; - p->from.offset = autosize; - p->from.reg = REGSP; - p->to.type = D_REG; - p->to.reg = REGPC; - // If there are instructions following - // this ARET, they come from a branch - // with the same stackframe, so no spadj. - } + p->as = AMOVW; + p->scond |= C_PBIT; + p->from.type = D_OREG; + p->from.offset = autosize; + p->from.reg = REGSP; + p->to.type = D_REG; + p->to.reg = REGPC; + // If there are instructions following + // this ARET, they come from a branch + // with the same stackframe, so no spadj. break; case AADD: @@ -589,7 +395,7 @@ noops(void) if(q1->reg == NREG) p->from.reg = q1->to.reg; p->to.type = D_REG; - p->to.reg = prog_div->from.sym->thumb ? REGTMPT : REGTMP; + p->to.reg = REGTMP; p->to.offset = 0; /* CALL appropriate */ @@ -598,14 +404,7 @@ noops(void) p->link = q; p = q; -#ifdef CALLEEBX p->as = ABL; -#else - if(prog_div->from.sym->thumb) - p->as = thumb ? ABL : ABX; - else - p->as = thumb ? ABX : ABL; -#endif p->line = q1->line; p->to.type = D_BRANCH; p->cond = p; @@ -637,7 +436,7 @@ noops(void) p->as = AMOVW; p->line = q1->line; p->from.type = D_REG; - p->from.reg = prog_div->from.sym->thumb ? REGTMPT : REGTMP; + p->from.reg = REGTMP; p->from.offset = 0; p->to.type = D_REG; p->to.reg = q1->to.reg; @@ -669,12 +468,6 @@ noops(void) break; case AMOVW: - if(thumb){ - Adr *a = &p->from; - - if(a->type == D_CONST && ((a->name == D_NONE && a->reg == REGSP) || a->name == D_AUTO || a->name == D_PARAM) && (a->offset & 3)) - diag("SP offset not multiple of 4"); - } if((p->scond & C_WBIT) && p->to.type == D_OREG && p->to.reg == REGSP) p->spadj = -p->to.offset; if((p->scond & C_PBIT) && p->from.type == D_OREG && p->from.reg == REGSP && p->to.reg != REGPC) @@ -682,136 +475,6 @@ noops(void) if(p->from.type == D_CONST && p->from.reg == REGSP && p->to.type == D_REG && p->to.reg == REGSP) p->spadj = -p->from.offset; break; - case AMOVB: - case AMOVBU: - case AMOVH: - case AMOVHU: - if(thumb){ - if(p->from.type == D_OREG && (p->from.name == D_AUTO || p->from.name == D_PARAM || (p->from.name == D_CONST && p->from.reg == REGSP))){ - q = prg(); - *q = *p; - if(p->from.name == D_AUTO) - q->from.offset += autosize; - else if(p->from.name == D_PARAM) - q->from.offset += autosize+4; - q->from.name = D_NONE; - q->from.reg = REGTMPT; - p = movrr(p, REGSP, REGTMPT, p); - q->link = p->link; - p->link = q; - } - if(p->to.type == D_OREG && (p->to.name == D_AUTO || p->to.name == D_PARAM || (p->to.name == D_CONST && p->to.reg == REGSP))){ - q = prg(); - *q = *p; - if(p->to.name == D_AUTO) - q->to.offset += autosize; - else if(p->to.name == D_PARAM) - q->to.offset += autosize+4; - q->to.name = D_NONE; - q->to.reg = REGTMPT; - p = movrr(p, REGSP, REGTMPT, p); - q->link = p->link; - p->link = q; - if(q->to.offset < 0 || q->to.offset > 255){ // complicated - p->to.reg = REGTMPT+1; // mov sp, r8 - q1 = prg(); - q1->line = p->line; - q1->as = AMOVW; - q1->from.type = D_CONST; - q1->from.offset = q->to.offset; - q1->to.type = D_REG; - q1->to.reg = REGTMPT; // mov $o, r7 - p->link = q1; - q1->link = q; - q1 = prg(); - q1->line = p->line; - q1->as = AADD; - q1->from.type = D_REG; - q1->from.reg = REGTMPT+1; - q1->to.type = D_REG; - q1->to.reg = REGTMPT; // add r8, r7 - p->link->link = q1; - q1->link = q; - q->to.offset = 0; // mov* r, 0(r7) - /* phew */ - } - } - } - break; - case AMOVM: - if(thumb){ - if(p->from.type == D_OREG){ - if(p->from.offset == 0) - p->from.type = D_REG; - else - diag("non-zero AMOVM offset"); - } - else if(p->to.type == D_OREG){ - if(p->to.offset == 0) - p->to.type = D_REG; - else - diag("non-zero AMOVM offset"); - } - } - break; - case AB: - if(thumb && p->to.type == D_OREG){ - if(p->to.offset == 0){ - p->as = AMOVW; - p->from.type = D_REG; - p->from.reg = p->to.reg; - p->to.type = D_REG; - p->to.reg = REGPC; - } - else{ - p->as = AADD; - p->from.type = D_CONST; - p->from.offset = p->to.offset; - p->reg = p->to.reg; - p->to.type = D_REG; - p->to.reg = REGTMPT-1; - q = prg(); - q->as = AMOVW; - q->line = p->line; - q->from.type = D_REG; - q->from.reg = REGTMPT-1; - q->to.type = D_REG; - q->to.reg = REGPC; - q->link = p->link; - p->link = q; - } - } - if(seenthumb && !thumb && p->to.type == D_OREG && p->to.reg == REGLINK){ - // print("warn %s: b (R%d) assuming a return\n", cursym->name, p->to.reg); - p->as = ABXRET; - } - break; - case ABL: - case ABX: - if(thumb && p->to.type == D_OREG){ - if(p->to.offset == 0){ - p->as = o; - p->from.type = D_NONE; - p->to.type = D_REG; - } - else{ - p->as = AADD; - p->from.type = D_CONST; - p->from.offset = p->to.offset; - p->reg = p->to.reg; - p->to.type = D_REG; - p->to.reg = REGTMPT-1; - q = prg(); - q->as = o; - q->line = p->line; - q->from.type = D_NONE; - q->to.type = D_REG; - q->to.reg = REGTMPT-1; - q->link = p->link; - p->link = q; - } - } - break; } } } @@ -876,13 +539,19 @@ setdiv(int as) Prog *p = nil; switch(as){ - case ADIV: p = prog_div; break; - case ADIVU: p = prog_divu; break; - case AMOD: p = prog_mod; break; - case AMODU: p = prog_modu; break; + case ADIV: + p = prog_div; + break; + case ADIVU: + p = prog_divu; + break; + case AMOD: + p = prog_mod; + break; + case AMODU: + p = prog_modu; + break; } - if(thumb != p->from.sym->thumb) - p->from.sym->foreign = 1; } void diff --git a/src/cmd/5l/obj.c b/src/cmd/5l/obj.c index c4a2bfc3f..dd3a7329a 100644 --- a/src/cmd/5l/obj.c +++ b/src/cmd/5l/obj.c @@ -67,6 +67,7 @@ static char* linkername[] = { "runtime.softfloat", + "math.sqrtGoC", }; void @@ -219,7 +220,7 @@ main(int argc, char *argv[]) elfinit(); HEADR = ELFRESERVE; if(INITTEXT == -1) - INITTEXT = 0x8000 + HEADR; + INITTEXT = 0x10000 + HEADR; if(INITDAT == -1) INITDAT = 0; if(INITRND == -1) @@ -241,7 +242,6 @@ main(int argc, char *argv[]) zprg.from.reg = NREG; zprg.to = zprg.from; buildop(); - thumbbuildop(); // could build on demand histgen = 0; pc = 0; dtype = 4; @@ -285,10 +285,8 @@ main(int argc, char *argv[]) asmb(); undef(); - if(debug['c']){ - thumbcount(); + if(debug['c']) print("ARM size = %d\n", armsize); - } if(debug['v']) { Bprint(&bso, "%5.2f cpu time\n", cputime()); Bprint(&bso, "%d sizeof adr\n", sizeof(Adr)); @@ -405,14 +403,12 @@ nopout(Prog *p) p->to.type = D_NONE; } -static void puntfp(Prog *); - void ldobj1(Biobuf *f, char *pkg, int64 len, char *pn) { int32 ipc; Prog *p; - Sym *h[NSYM], *s, *di; + Sym *h[NSYM], *s; int v, o, r, skip; uint32 sig; char *name; @@ -424,7 +420,6 @@ ldobj1(Biobuf *f, char *pkg, int64 len, char *pn) lastp = nil; ntext = 0; eof = Boffset(f) + len; - di = S; src[0] = 0; newloop: @@ -622,8 +617,6 @@ loop: else textp = s; etextp = s; - setarch(p); - setthumb(p); p->align = 4; autosize = (p->to.offset+3L) & ~3L; p->to.offset = autosize; @@ -631,7 +624,6 @@ loop: s->type = STEXT; s->text = p; s->value = pc; - s->thumb = thumb; lastp = p; p->pc = pc; pc++; @@ -673,13 +665,9 @@ loop: case AMULD: case ADIVF: case ADIVD: - if(thumb) - puntfp(p); goto casedef; case AMOVF: - if(thumb) - puntfp(p); if(skip) goto casedef; @@ -701,8 +689,6 @@ loop: goto casedef; case AMOVD: - if(thumb) - puntfp(p); if(skip) goto casedef; @@ -758,17 +744,6 @@ prg(void) return p; } -static void -puntfp(Prog *p) -{ - USED(p); - /* floating point - punt for now */ - cursym->text->reg = NREG; /* ARM */ - cursym->thumb = 0; - thumb = 0; - // print("%s: generating ARM code (contains floating point ops %d)\n", curtext->from.sym->name, p->line); -} - Prog* appendp(Prog *q) { diff --git a/src/cmd/5l/optab.c b/src/cmd/5l/optab.c index 625b66812..514786f85 100644 --- a/src/cmd/5l/optab.c +++ b/src/cmd/5l/optab.c @@ -77,7 +77,6 @@ Optab optab[] = { ASWI, C_NONE, C_NONE, C_LCON, 10, 4, 0 }, { AWORD, C_NONE, C_NONE, C_LCON, 11, 4, 0 }, - { AWORD, C_NONE, C_NONE, C_GCON, 11, 4, 0 }, { AWORD, C_NONE, C_NONE, C_ADDR, 11, 4, 0 }, { AMOVW, C_NCON, C_NONE, C_REG, 12, 4, 0 }, @@ -117,18 +116,6 @@ Optab optab[] = { AMOVBU, C_SAUTO,C_NONE, C_REG, 21, 4, REGSP }, { AMOVBU, C_SOREG,C_NONE, C_REG, 21, 4, 0 }, - { AMOVB, C_SAUTO,C_NONE, C_REG, 22, 12, REGSP }, - { AMOVB, C_SOREG,C_NONE, C_REG, 22, 12, 0 }, - { AMOVH, C_SAUTO,C_NONE, C_REG, 22, 12, REGSP }, - { AMOVH, C_SOREG,C_NONE, C_REG, 22, 12, 0 }, - { AMOVHU, C_SAUTO,C_NONE, C_REG, 22, 12, REGSP }, - { AMOVHU, C_SOREG,C_NONE, C_REG, 22, 12, 0 }, - - { AMOVH, C_REG, C_NONE, C_SAUTO, 23, 12, REGSP }, - { AMOVH, C_REG, C_NONE, C_SOREG, 23, 12, 0 }, - { AMOVHU, C_REG, C_NONE, C_SAUTO, 23, 12, REGSP }, - { AMOVHU, C_REG, C_NONE, C_SOREG, 23, 12, 0 }, - { AMOVW, C_REG, C_NONE, C_LAUTO, 30, 8, REGSP, LTO }, { AMOVW, C_REG, C_NONE, C_LOREG, 30, 8, 0, LTO }, { AMOVW, C_REG, C_NONE, C_ADDR, 64, 8, 0, LTO }, @@ -146,23 +133,6 @@ Optab optab[] = { AMOVBU, C_LOREG,C_NONE, C_REG, 31, 8, 0, LFROM }, { AMOVBU, C_ADDR, C_NONE, C_REG, 65, 8, 0, LFROM }, - { AMOVB, C_LAUTO,C_NONE, C_REG, 32, 16, REGSP, LFROM }, - { AMOVB, C_LOREG,C_NONE, C_REG, 32, 16, 0, LFROM }, - { AMOVB, C_ADDR, C_NONE, C_REG, 66, 16, 0, LFROM }, - { AMOVH, C_LAUTO,C_NONE, C_REG, 32, 16, REGSP, LFROM }, - { AMOVH, C_LOREG,C_NONE, C_REG, 32, 16, 0, LFROM }, - { AMOVH, C_ADDR, C_NONE, C_REG, 66, 16, 0, LFROM }, - { AMOVHU, C_LAUTO,C_NONE, C_REG, 32, 16, REGSP, LFROM }, - { AMOVHU, C_LOREG,C_NONE, C_REG, 32, 16, 0, LFROM }, - { AMOVHU, C_ADDR, C_NONE, C_REG, 66, 16, 0, LFROM }, - - { AMOVH, C_REG, C_NONE, C_LAUTO, 33, 24, REGSP, LTO }, - { AMOVH, C_REG, C_NONE, C_LOREG, 33, 24, 0, LTO }, - { AMOVH, C_REG, C_NONE, C_ADDR, 67, 24, 0, LTO }, - { AMOVHU, C_REG, C_NONE, C_LAUTO, 33, 24, REGSP, LTO }, - { AMOVHU, C_REG, C_NONE, C_LOREG, 33, 24, 0, LTO }, - { AMOVHU, C_REG, C_NONE, C_ADDR, 67, 24, 0, LTO }, - { AMOVW, C_LACON,C_NONE, C_REG, 34, 8, REGSP, LFROM }, { AMOVW, C_PSR, C_NONE, C_REG, 35, 4, 0 }, @@ -210,29 +180,35 @@ Optab optab[] = { ACASE, C_REG, C_NONE, C_NONE, 62, 4, 0 }, { ABCASE, C_NONE, C_NONE, C_SBRA, 63, 4, 0 }, - { AMOVH, C_REG, C_NONE, C_HAUTO, 70, 4, REGSP, V4 }, - { AMOVH, C_REG, C_NONE, C_HOREG, 70, 4, 0, V4 }, - { AMOVHU, C_REG, C_NONE, C_HAUTO, 70, 4, REGSP, V4 }, - { AMOVHU, C_REG, C_NONE, C_HOREG, 70, 4, 0, V4 }, - - { AMOVB, C_HAUTO,C_NONE, C_REG, 71, 4, REGSP, V4 }, - { AMOVB, C_HOREG,C_NONE, C_REG, 71, 4, 0, V4 }, - { AMOVH, C_HAUTO,C_NONE, C_REG, 71, 4, REGSP, V4 }, - { AMOVH, C_HOREG,C_NONE, C_REG, 71, 4, 0, V4 }, - { AMOVHU, C_HAUTO,C_NONE, C_REG, 71, 4, REGSP, V4 }, - { AMOVHU, C_HOREG,C_NONE, C_REG, 71, 4, 0, V4 }, - - { AMOVH, C_REG, C_NONE, C_LAUTO, 72, 8, REGSP, LTO|V4 }, - { AMOVH, C_REG, C_NONE, C_LOREG, 72, 8, 0, LTO|V4 }, - { AMOVHU, C_REG, C_NONE, C_LAUTO, 72, 8, REGSP, LTO|V4 }, - { AMOVHU, C_REG, C_NONE, C_LOREG, 72, 8, 0, LTO|V4 }, - - { AMOVB, C_LAUTO,C_NONE, C_REG, 73, 8, REGSP, LFROM|V4 }, - { AMOVB, C_LOREG,C_NONE, C_REG, 73, 8, 0, LFROM|V4 }, - { AMOVH, C_LAUTO,C_NONE, C_REG, 73, 8, REGSP, LFROM|V4 }, - { AMOVH, C_LOREG,C_NONE, C_REG, 73, 8, 0, LFROM|V4 }, - { AMOVHU, C_LAUTO,C_NONE, C_REG, 73, 8, REGSP, LFROM|V4 }, - { AMOVHU, C_LOREG,C_NONE, C_REG, 73, 8, 0, LFROM|V4 }, + { AMOVH, C_REG, C_NONE, C_HAUTO, 70, 4, REGSP, 0 }, + { AMOVH, C_REG, C_NONE, C_HOREG, 70, 4, 0, 0 }, + { AMOVHU, C_REG, C_NONE, C_HAUTO, 70, 4, REGSP, 0 }, + { AMOVHU, C_REG, C_NONE, C_HOREG, 70, 4, 0, 0 }, + + { AMOVB, C_HAUTO,C_NONE, C_REG, 71, 4, REGSP, 0 }, + { AMOVB, C_HOREG,C_NONE, C_REG, 71, 4, 0, 0 }, + { AMOVH, C_HAUTO,C_NONE, C_REG, 71, 4, REGSP, 0 }, + { AMOVH, C_HOREG,C_NONE, C_REG, 71, 4, 0, 0 }, + { AMOVHU, C_HAUTO,C_NONE, C_REG, 71, 4, REGSP, 0 }, + { AMOVHU, C_HOREG,C_NONE, C_REG, 71, 4, 0, 0 }, + + { AMOVH, C_REG, C_NONE, C_LAUTO, 72, 8, REGSP, LTO }, + { AMOVH, C_REG, C_NONE, C_LOREG, 72, 8, 0, LTO }, + { AMOVH, C_REG, C_NONE, C_ADDR, 94, 8, 0, LTO }, + { AMOVHU, C_REG, C_NONE, C_LAUTO, 72, 8, REGSP, LTO }, + { AMOVHU, C_REG, C_NONE, C_LOREG, 72, 8, 0, LTO }, + { AMOVHU, C_REG, C_NONE, C_ADDR, 94, 8, 0, LTO }, + + { AMOVB, C_LAUTO,C_NONE, C_REG, 73, 8, REGSP, LFROM }, + { AMOVB, C_LOREG,C_NONE, C_REG, 73, 8, 0, LFROM }, + { AMOVB, C_ADDR, C_NONE, C_REG, 93, 8, 0, LFROM }, + { AMOVH, C_LAUTO,C_NONE, C_REG, 73, 8, REGSP, LFROM }, + { AMOVH, C_LOREG,C_NONE, C_REG, 73, 8, 0, LFROM }, + { AMOVH, C_ADDR, C_NONE, C_REG, 93, 8, 0, LFROM }, + { AMOVHU, C_LAUTO,C_NONE, C_REG, 73, 8, REGSP, LFROM }, + { AMOVHU, C_LOREG,C_NONE, C_REG, 73, 8, 0, LFROM }, + { AMOVHU, C_ADDR, C_NONE, C_REG, 93, 8, 0, LFROM }, + { ALDREX, C_SOREG,C_NONE, C_REG, 77, 4, 0 }, { ASTREX, C_SOREG,C_REG, C_REG, 78, 4, 0 }, diff --git a/src/cmd/5l/pass.c b/src/cmd/5l/pass.c index 7e1ba6a09..194a1ed5f 100644 --- a/src/cmd/5l/pass.c +++ b/src/cmd/5l/pass.c @@ -100,7 +100,6 @@ xfol(Prog *p, Prog **last) loop: if(p == P) return; - setarch(p); a = p->as; if(a == AB) { q = p->cond; @@ -210,14 +209,7 @@ patch(void) vexit = s->value; for(cursym = textp; cursym != nil; cursym = cursym->next) { for(p = cursym->text; p != P; p = p->link) { - setarch(p); a = p->as; - if(seenthumb && a == ABL){ - // if((s = p->to.sym) != S && (s1 = curtext->from.sym) != S) - // print("%s calls %s\n", s1->name, s->name); - if((s = p->to.sym) != S && s->thumb != cursym->thumb) - s->foreign = 1; - } if((a == ABL || a == ABX || a == AB || a == ARET) && p->to.type != D_BRANCH && p->to.sym != S) { s = p->to.sym; @@ -254,19 +246,7 @@ patch(void) for(cursym = textp; cursym != nil; cursym = cursym->next) { for(p = cursym->text; p != P; p = p->link) { - setarch(p); a = p->as; - if(seenthumb && a == ABL) { -#ifdef CALLEEBX - if(0) - {} -#else - if((s = p->to.sym) != S && (s->foreign || s->fnptr)) - p->as = ABX; -#endif - else if(p->to.type == D_OREG) - p->as = ABX; - } if(p->cond != P) { p->cond = brloop(p->cond); if(p->cond != P) diff --git a/src/cmd/5l/prof.c b/src/cmd/5l/prof.c index ad115a8ca..48ad2dc59 100644 --- a/src/cmd/5l/prof.c +++ b/src/cmd/5l/prof.c @@ -47,7 +47,6 @@ doprof1(void) s = lookup("__mcount", 0); n = 1; for(p = firstp->link; p != P; p = p->link) { - setarch(p); if(p->as == ATEXT) { q = prg(); q->line = p->line; @@ -74,7 +73,7 @@ doprof1(void) p->from.sym = s; p->from.offset = n*4 + 4; p->to.type = D_REG; - p->to.reg = thumb ? REGTMPT : REGTMP; + p->to.reg = REGTMP; q = prg(); q->line = p->line; @@ -86,7 +85,7 @@ doprof1(void) p->from.type = D_CONST; p->from.offset = 1; p->to.type = D_REG; - p->to.reg = thumb ? REGTMPT : REGTMP; + p->to.reg = REGTMP; q = prg(); q->line = p->line; @@ -96,7 +95,7 @@ doprof1(void) p = q; p->as = AMOVW; p->from.type = D_REG; - p->from.reg = thumb ? REGTMPT : REGTMP; + p->from.reg = REGTMP; p->to.type = D_OREG; p->to.name = D_EXTERN; p->to.sym = s; @@ -143,7 +142,6 @@ doprof2(void) ps4 = P; for(cursym = textp; cursym != nil; cursym = cursym->next) { p = cursym->text; - setarch(p); if(cursym == s2) { ps2 = p; p->reg = 1; @@ -155,7 +153,6 @@ doprof2(void) } for(cursym = textp; cursym != nil; cursym = cursym->next) for(p = cursym->text; p != P; p = p->link) { - setarch(p); if(p->as == ATEXT) { if(p->reg & NOPROF) { for(;;) { diff --git a/src/cmd/5l/softfloat.c b/src/cmd/5l/softfloat.c index 03d8c6d26..4f799d17e 100644 --- a/src/cmd/5l/softfloat.c +++ b/src/cmd/5l/softfloat.c @@ -54,6 +54,8 @@ softfloat(void) case AMULD: case ADIVF: case ADIVD: + case ASQRTF: + case ASQRTD: goto soft; default: diff --git a/src/cmd/5l/span.c b/src/cmd/5l/span.c index 482d3e90a..eb79f6b5a 100644 --- a/src/cmd/5l/span.c +++ b/src/cmd/5l/span.c @@ -50,86 +50,6 @@ isbranch(Prog *p) } static int -ispad(Prog *p) -{ - if(p->as != AMOVW) - return 0; - if(p->from.type != D_REG || p->from.reg != REGTMP) - return 0; - if(p->to.type != D_REG || p->to.reg != REGTMP) - return 0; - return 1; -} - -int -fninc(Sym *s) -{ - if(thumb){ - if(s->thumb){ - if(s->foreign) - return 8; - else - return 0; - } - else{ - if(s->foreign) - return 0; - else - diag("T A !foreign in fninc"); - } - } - else{ - if(s->thumb){ - if(s->foreign) - return 0; - else - diag("A T !foreign in fninc"); - } - else{ - if(s->foreign) - return 4; - else - return 0; - } - } - return 0; -} - -int -fnpinc(Sym *s) -{ - if(!s->fnptr){ // a simplified case BX O(R) -> BL O(R) - if(!debug['f']) - diag("fnptr == 0 in fnpinc"); - if(s->foreign) - diag("bad usage in fnpinc %s %d %d", s->name, s->foreign, s->thumb); - return 0; - } - /* 0, 1, 2, 3 squared */ - if(s->thumb) - return s->foreign ? 9 : 1; - else - return s->foreign ? 4 : 0; -} - -static Prog * -pad(Prog *p, int pc) -{ - Prog *q; - - q = prg(); - q->as = AMOVW; - q->line = p->line; - q->from.type = D_REG; - q->from.reg = REGTMP; - q->to.type = D_REG; - q->to.reg = REGTMP; - q->pc = pc; - q->link = p->link; - return q; -} - -static int scan(Prog *op, Prog *p, int c) { Prog *q; @@ -168,7 +88,6 @@ span(void) Optab *o; int m, bflag, i, v; int32 c, otxt, out[6]; - int lastthumb = -1; Section *sect; uchar *bp; @@ -183,11 +102,9 @@ span(void) otxt = c; for(cursym = textp; cursym != nil; cursym = cursym->next) { p = cursym->text; - setarch(p); p->pc = c; cursym->value = c; - lastthumb = thumb; autosize = p->to.offset + 4; if(p->from.sym != S) p->from.sym->value = c; @@ -195,19 +112,14 @@ span(void) if(c-otxt >= 1L<<17) bflag = 1; otxt = c; - if(thumb && blitrl) - pool.extra += brextra(p); for(op = p, p = p->link; p != P; op = p, p = p->link) { curp = p; - setarch(p); p->pc = c; o = oplook(p); m = o->size; // must check literal pool here in case p generates many instructions if(blitrl){ - if(thumb && isbranch(p)) - pool.extra += brextra(p); if(checkpool(op, p->as == ACASE ? casesz(p) : m)) c = p->pc = scan(op, p, c); } @@ -232,8 +144,6 @@ span(void) c += m; } if(blitrl){ - if(thumb && isbranch(op)) - pool.extra += brextra(op); if(checkpool(op, 0)) c = scan(op, P, c); } @@ -255,10 +165,7 @@ span(void) cursym->value = c; for(p = cursym->text; p != P; p = p->link) { curp = p; - setarch(p); p->pc = c; - if(thumb && isbranch(p)) - nocache(p); o = oplook(p); /* very large branches if(o->type == 6 && p->cond) { @@ -300,74 +207,6 @@ span(void) } } - if(seenthumb){ // branch resolution - int passes = 0; - int lastc = 0; - int again; - Prog *oop; - - loop: - passes++; - if(passes > 100){ - diag("span looping !"); - errorexit(); - } - c = INITTEXT; - oop = op = nil; - again = 0; - for(cursym = textp; cursym != nil; cursym = cursym->next) { - cursym->value = c; - for(p = cursym->text; p != P; oop = op, op = p, p = p->link) { - curp = p; - setarch(p); - if(p->pc != c) - again = 1; - p->pc = c; - if(thumb && isbranch(p)) - nocache(p); - o = oplook(p); - m = o->size; - if(passes == 1 && thumb && isbranch(p)){ // start conservative so unneeded alignment is not added - if(p->as == ABL) - m = 4; - else - m = 2; - p->align = 0; - } - if(p->align){ - if((p->align == 4 && (c&3)) || (p->align == 2 && !(c&3))){ - if(ispad(op)){ - oop->link = p; - op = oop; - c -= 2; - p->pc = c; - } - else{ - op->link = pad(op, c); - op = op->link; - c += 2; - p->pc = c; - } - again = 1; - } - } - if(m == 0) { - if(p->as == ATEXT) { - autosize = p->to.offset + 4; - if(p->from.sym != S) - p->from.sym->value = c; - continue; - } - } - c += m; - } - cursym->size = c - cursym->value; - } - if(c != lastc || again){ - lastc = c; - goto loop; - } - } c = rnd(c, 8); /* @@ -380,7 +219,6 @@ span(void) */ for(cursym = textp; cursym != nil; cursym = cursym->next) { p = cursym->text; - setarch(p); autosize = p->to.offset + 4; symgrow(cursym, cursym->size); @@ -414,13 +252,6 @@ span(void) int checkpool(Prog *p, int sz) { - if(thumb){ - if(pool.size >= 0x3fc || (p->pc+sz+pool.extra+2+2)+(pool.size-4)-pool.start-4 >= 0x3fc) - return flushpool(p, 1, 0); - else if(p->link == P) - return flushpool(p, 2, 0); - return 0; - } if(pool.size >= 0xffc || immaddr((p->pc+sz+4)+4+pool.size - pool.start+8) == 0) return flushpool(p, 1, 0); else if(p->link == P) @@ -443,7 +274,7 @@ flushpool(Prog *p, int skip, int force) q->link = blitrl; blitrl = q; } - else if(!force && (p->pc+pool.size-pool.start < (thumb ? 0x3fc+4-pool.extra : 2048))) + else if(!force && (p->pc+pool.size-pool.start < 2048)) return 0; elitrl->link = p->link; p->link = blitrl; @@ -463,10 +294,7 @@ addpool(Prog *p, Adr *a) Prog *q, t; int c; - if(thumb) - c = thumbaclass(a, p); - else - c = aclass(a); + c = aclass(a); t = zprg; t.as = AWORD; @@ -482,12 +310,10 @@ addpool(Prog *p, Adr *a) case C_FOREG: case C_SOREG: case C_HOREG: - case C_GOREG: case C_FAUTO: case C_SAUTO: case C_LAUTO: case C_LACON: - case C_GACON: t.to.type = D_CONST; t.to.offset = instoffset; break; @@ -593,16 +419,6 @@ symaddr(Sym *s) return 0; case STEXT: -/* TODO(rsc): what is this for? -#ifdef CALLEEBX - v += fnpinc(s); -#else - if(s->thumb) - v++; // T bit -#endif -*/ - break; - case SELFDATA: case SRODATA: case SDATA: @@ -770,35 +586,19 @@ oplook(Prog *p) int a1, a2, a3, r; char *c1, *c3; Optab *o, *e; - Optab *otab; - Oprang *orange; - if(thumb){ - otab = thumboptab; - orange = thumboprange; - } - else{ - otab = optab; - orange = oprange; - } a1 = p->optab; if(a1) - return otab+(a1-1); + return optab+(a1-1); a1 = p->from.class; if(a1 == 0) { - if(thumb) - a1 = thumbaclass(&p->from, p) + 1; - else - a1 = aclass(&p->from) + 1; + a1 = aclass(&p->from) + 1; p->from.class = a1; } a1--; a3 = p->to.class; if(a3 == 0) { - if(thumb) - a3 = thumbaclass(&p->to, p) + 1; - else - a3 = aclass(&p->to) + 1; + a3 = aclass(&p->to) + 1; p->to.class = a3; } a3--; @@ -806,35 +606,35 @@ oplook(Prog *p) if(p->reg != NREG) a2 = C_REG; r = p->as; - o = orange[r].start; + o = oprange[r].start; if(o == 0) { a1 = opcross[repop[r]][a1][a2][a3]; if(a1) { p->optab = a1+1; - return otab+a1; + return optab+a1; } - o = orange[r].stop; /* just generate an error */ + o = oprange[r].stop; /* just generate an error */ } if(debug['O']) { print("oplook %A %O %O %O\n", (int)p->as, a1, a2, a3); print(" %d %d\n", p->from.type, p->to.type); } - e = orange[r].stop; + e = oprange[r].stop; c1 = xcmp[a1]; c3 = xcmp[a3]; for(; o<e; o++) if(o->a2 == a2) if(c1[o->a1]) if(c3[o->a3]) { - p->optab = (o-otab)+1; + p->optab = (o-optab)+1; return o; } diag("illegal combination %A %O %O %O, %d %d", p->as, a1, a2, a3, p->from.type, p->to.type); prasm(p); if(o == 0) - o = otab; + o = optab; return o; } @@ -885,9 +685,6 @@ cmp(int a, int b) if(b == C_SBRA) return 1; break; - case C_GBRA: - if(b == C_SBRA || b == C_LBRA) - return 1; case C_HREG: return cmp(C_SP, b) || cmp(C_PC, b); @@ -907,9 +704,6 @@ ocmp(const void *a1, const void *a2) n = p1->as - p2->as; if(n) return n; - n = (p2->flag&V4) - (p1->flag&V4); /* architecture version */ - if(n) - return n; n = p1->a1 - p2->a1; if(n) return n; @@ -927,15 +721,11 @@ buildop(void) { int i, n, r; - armv4 = !debug['h']; for(i=0; i<C_GOK; i++) for(n=0; n<C_GOK; n++) xcmp[i][n] = cmp(n, i); for(n=0; optab[n].as != AXXX; n++) - if((optab[n].flag & V4) && !armv4) { - optab[n].as = AXXX; - break; - } + ; qsort(optab, n, sizeof(optab[0]), ocmp); for(i=0; i<n; i++) { r = optab[i].as; @@ -1025,6 +815,8 @@ buildop(void) oprange[AMULD] = oprange[r]; oprange[ADIVF] = oprange[r]; oprange[ADIVD] = oprange[r]; + oprange[ASQRTF] = oprange[r]; + oprange[ASQRTD] = oprange[r]; oprange[AMOVFD] = oprange[r]; oprange[AMOVDF] = oprange[r]; break; diff --git a/src/cmd/5l/thumb.c b/src/cmd/5l/thumb.c deleted file mode 100644 index a6f729bed..000000000 --- a/src/cmd/5l/thumb.c +++ /dev/null @@ -1,1658 +0,0 @@ -// Inferno utils/5l/thumb.c -// http://code.google.com/p/inferno-os/source/browse/utils/5l/thumb.c -// -// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. -// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) -// Portions Copyright © 1997-1999 Vita Nuova Limited -// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) -// Portions Copyright © 2004,2006 Bruce Ellis -// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) -// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others -// Portions Copyright © 2009 The Go Authors. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#include "l.h" -#include "../ld/lib.h" - -static int32 thumboprr(int); -static int32 thumboprrr(int, int); -static int32 thumbopirr(int , int); -static int32 thumbopri(int); -static int32 thumbophh(int); -static int32 thumbopbra(int); -static int32 thumbopmv(int, int); -static void lowreg(Prog *, int); -static void mult(Prog *, int, int); -static void numr(Prog *, int, int, int); -static void regis(Prog *, int, int, int); -static void dis(int, int); - -// build a constant using neg, add and shift - only worth it if < 6 bytes */ -static int -immbuildcon(int c, Prog *p) -{ - int n = 0; - - USED(p); - if(c >= 0 && c <= 255) - return 0; // mv - if(c >= -255 && c < 0) // mv, neg - return 1; - if(c >= 256 && c <= 510) // mv, add - return 1; - if(c < 0) - return 0; - while(!(c & 1)){ - n++; - c >>= 1; - } - if(c >= 0 && c <= 255) // mv, lsl - return 1; - return 0; -} - -// positive 5 bit offset from register - O(R) -// positive 8 bit offset from register - mov O, R then [R, R] -// otherwise O goes in literal pool - mov O1(PC), R then [R, R] -static int -immoreg(int off, Prog *p) -{ - int v = 1; - int as = p->as; - - if(off < 0) - return C_GOREG; - if(as == AMOVW) - v = 4; - else if(as == AMOVH || as == AMOVHU) - v = 2; - else if(as == AMOVB || as == AMOVBU) - v = 1; - else - diag("bad op in immoreg"); - if(off/v <= 31) - return C_SOREG; - if(off <= 255) - return C_LOREG; - return C_GOREG; -} - -// positive 8 bit - mov O, R then 0(R) -// otherwise O goes in literal pool - mov O1(PC), R then 0(R) -static int -immacon(int off, Prog *p, int t1, int t2) -{ - USED(p); - if(off < 0) - return t2; - if(off <= 255) - return t1; - return t2; -} - -// unsigned 8 bit in words -static int -immauto(int off, Prog *p) -{ - if(p->as != AMOVW) - diag("bad op in immauto"); - mult(p, off, 4); - if(off >= 0 && off <= 1020) - return C_SAUTO; - return C_LAUTO; -} - -static int -immsmall(int off, Prog *p, int t1, int t2, int t3) -{ - USED(p); - if(off >= 0 && off <= 7) - return t1; - if(off >= 0 && off <= 255) - return t2; - return t3; -} - -static int -immcon(int off, Prog *p) -{ - int as = p->as; - - if(as == ASLL || as == ASRL || as == ASRA) - return C_SCON; - if(p->to.type == D_REG && p->to.reg == REGSP){ - if(as == AADD || as == ASUB){ - if(off >= 0 && off <= 508) - return C_SCON; - if(as == ASUB){ - p->as = AADD; - p->from.offset = -p->from.offset; - } - return C_LCON; - } - diag("unknown type in immcon"); - } - if(as == AADD || as == ASUB){ - if(p->reg != NREG) - return immsmall(off, p, C_SCON, C_LCON, C_GCON); - return immacon(off, p, C_SCON, C_LCON); - } - if(as == AMOVW && p->from.type == D_CONST && p->to.type == D_REG && immbuildcon(off, p)) - return C_BCON; - if(as == ACMP && p->from.type == D_CONST && immbuildcon(off, p)) - return C_BCON; - if(as == ACMP || as == AMOVW) - return immacon(off, p, C_SCON, C_LCON); - return C_LCON; -} - -int -thumbaclass(Adr *a, Prog *p) -{ - Sym *s; - int t; - - switch(a->type) { - case D_NONE: - return C_NONE; - case D_REG: - if(a->reg == REGSP) - return C_SP; - if(a->reg == REGPC) - return C_PC; - if(a->reg >= 8) - return C_HREG; - return C_REG; - case D_SHIFT: - diag("D_SHIFT in thumbaclass"); - return C_SHIFT; - case D_FREG: - diag("D_FREG in thumbaclass"); - return C_FREG; - case D_FPCR: - diag("D_FPCR in thumbaclass"); - return C_FCR; - case D_OREG: - switch(a->name) { - case D_EXTERN: - case D_STATIC: - if(a->sym == 0 || a->sym->name == 0) { - print("null sym external\n"); - print("%D\n", a); - return C_GOK; - } - t = a->sym->type; - if(t == 0 || t == SXREF) { - diag("undefined external: %s in %s\n", - a->sym->name, TNAME); - a->sym->type = SDATA; - } - instoffset = a->sym->value + a->offset; - return C_ADDR; /* INITDAT unknown at this stage */ - case D_AUTO: - instoffset = autosize + a->offset; - return immauto(instoffset, p); - case D_PARAM: - instoffset = autosize + a->offset + 4L; -// print("D_PARAM %s %d+%d+%d = %d\n", a->sym != S ? a->sym->name : "noname", autosize, a->offset, 4, autosize+a->offset+4); - return immauto(instoffset, p); - case D_NONE: - instoffset = a->offset; - if(a->reg == REGSP) - return immauto(instoffset, p); - else - return immoreg(instoffset, p); - } - return C_GOK; - case D_PSR: - diag("D_PSR in thumbaclass"); - return C_PSR; - case D_OCONST: - switch(a->name) { - case D_EXTERN: - case D_STATIC: - s = a->sym; - t = s->type; - if(t == 0 || t == SXREF) { - diag("undefined external: %s in %s\n", - s->name, TNAME); - s->type = SDATA; - } - instoffset = s->value + a->offset; - if(s->type == STEXT){ - instoffset = s->value + a->offset; -#ifdef CALLEEBX - instoffset += fnpinc(s); -#else - if(s->thumb) - instoffset++; // T bit -#endif - return C_LCON; - } - return C_LCON; /* INITDAT unknown at this stage */ - // return immcon(instoffset, p); - } - return C_GOK; - case D_FCONST: - diag("D_FCONST in thumaclass"); - return C_LFCON; - case D_CONST: - switch(a->name) { - case D_NONE: - instoffset = a->offset; - if(a->reg != NREG) - goto aconsize; - return immcon(instoffset, p); - case D_EXTERN: - case D_STATIC: - s = a->sym; - if(s == S) - break; - t = s->type; - switch(t) { - case 0: - case SXREF: - diag("undefined external: %s in %s\n", - s->name, TNAME); - s->type = SDATA; - break; - case SCONST: - case STEXT: - instoffset = s->value + a->offset; -#ifdef CALLEEBX - instoffset += fnpinc(s); -#else - if(s->thumb) - instoffset++; // T bit -#endif - return C_LCON; - } - instoffset = s->value + a->offset; - return C_LCON; /* INITDAT unknown at this stage */ - // return immcon(instoffset, p); - case D_AUTO: - instoffset = autosize + a->offset; - goto aconsize; - case D_PARAM: - instoffset = autosize + a->offset + 4L; - aconsize: - if(p->from.reg == REGSP || p->from.reg == NREG) - return instoffset >= 0 && instoffset < 1024 ? C_SACON : C_GACON; - else if(p->from.reg == p->to.reg) - return immacon(instoffset, p, C_SACON, C_GACON); - return immsmall(instoffset, p, C_SACON, C_LACON, C_GACON); - } - return C_GOK; - case D_BRANCH: { - int v, va; - - p->align = 0; - v = -4; - va = 0; - if(p->cond != P){ - v = (p->cond->pc - p->pc) - 4; - va = p->cond->pc; - } - instoffset = v; - if(p->as == AB){ - if(v >= -2048 && v <= 2046) - return C_SBRA; - p->align = 4; - instoffset = va; - return C_LBRA; - } - if(p->as == ABL){ -#ifdef CALLEEBX - int e; - - if((e = fninc(p->to.sym))) { - v += e; - va += e; - instoffset += e; - } -#endif - if(v >= -4194304 && v <= 4194302) - return C_SBRA; - p->align = 2; - instoffset = va; - return C_LBRA; - } - if(p->as == ABX){ - v = va; - if(v >= 0 && v <= 255) - return C_SBRA; - p->align = 2; - instoffset = va; - return C_LBRA; - } - if(v >= -256 && v <= 254) - return C_SBRA; - if(v >= -(2048-2) && v <= (2046+2)) - return C_LBRA; - p->align = 2; - instoffset = va; - return C_GBRA; - } - } - return C_GOK; -} - -// as a1 a2 a3 type size param lit vers -Optab thumboptab[] = -{ - { ATEXT, C_ADDR, C_NONE, C_LCON, 0, 0, 0 }, - { ATEXT, C_ADDR, C_REG, C_LCON, 0, 0, 0 }, - { AMVN, C_REG, C_NONE, C_REG, 1, 2, 0 }, - { ASRL, C_REG, C_NONE, C_REG, 1, 2, 0 }, - { ACMP, C_REG, C_REG, C_NONE, 1, 2, 0 }, - { ACMN, C_REG, C_REG, C_NONE, 1, 2, 0 }, - { AADD, C_REG, C_REG, C_REG, 2, 2, 0 }, - { AADD, C_REG, C_NONE, C_REG, 2, 2, 0 }, - { AADD, C_SCON, C_REG, C_REG, 3, 2, 0 }, - { AADD, C_LCON, C_REG, C_REG, 49, 4, 0 }, - { AADD, C_GCON, C_REG, C_REG, 36, 4, 0, LFROM }, - // { AADD, C_LCON, C_NONE, C_REG, 3, 2, 0, LFROM }, - { ASRL, C_SCON, C_REG, C_REG, 4, 2, 0 }, - { ASRL, C_SCON, C_NONE, C_REG, 4, 2, 0 }, - { AADD, C_SCON, C_NONE, C_REG, 5, 2, 0 }, - { AADD, C_LCON, C_NONE, C_REG, 37, 4, 0, LFROM }, - { ACMP, C_SCON, C_REG, C_NONE, 5, 2, 0 }, - { ACMP, C_BCON, C_REG, C_NONE, 48, 6, 0 }, - { ACMP, C_LCON, C_REG, C_NONE, 39, 4, 0, LFROM }, - { AMOVW, C_SCON, C_NONE, C_REG, 5, 2, 0 }, - { AMOVW, C_BCON, C_NONE, C_REG, 47, 4, 0 }, - { AMOVW, C_LCON, C_NONE, C_REG, 38, 2, 0, LFROM }, - // { AADD, C_LCON, C_PC, C_REG, 6, 2, 0, LFROM }, - // { AADD, C_LCON, C_SP, C_REG, 6, 2, 0, LFROM }, - { AADD, C_SCON, C_NONE, C_SP, 7, 2, 0 }, - { AADD, C_LCON, C_NONE, C_SP, 40, 4, 0, LFROM }, - { AADD, C_REG, C_NONE, C_HREG, 8, 2, 0 }, - { AADD, C_HREG, C_NONE, C_REG, 8, 2, 0 }, - { AADD, C_HREG, C_NONE, C_HREG, 8, 2, 0 }, - { AMOVW, C_REG, C_NONE, C_HREG, 8, 2, 0 }, - { AMOVW, C_HREG, C_NONE, C_REG, 8, 2, 0 }, - { AMOVW, C_HREG, C_NONE, C_HREG, 8, 2, 0 }, - { ACMP, C_REG, C_HREG, C_NONE, 8, 2, 0 }, - { ACMP, C_HREG, C_REG, C_NONE, 8, 2, 0 }, - { ACMP, C_HREG, C_HREG, C_NONE, 8, 2, 0 }, - { AB, C_NONE, C_NONE, C_SBRA, 9, 2, 0, LPOOL }, - { ABEQ, C_NONE, C_NONE, C_SBRA, 10, 2, 0 }, - { ABL, C_NONE, C_NONE, C_SBRA, 11, 4, 0 }, - { ABX, C_NONE, C_NONE, C_SBRA, 12, 10, 0 }, - { AB, C_NONE, C_NONE, C_LBRA, 41, 8, 0, LPOOL }, - { ABEQ, C_NONE, C_NONE, C_LBRA, 46, 4, 0 }, - { ABL, C_NONE, C_NONE, C_LBRA, 43, 14, 0 }, - { ABX, C_NONE, C_NONE, C_LBRA, 44, 14, 0 }, - { ABEQ, C_NONE, C_NONE, C_GBRA, 42, 10, 0 }, - // { AB, C_NONE, C_NONE, C_SOREG, 13, 0, 0 }, - // { ABL, C_NONE, C_NONE, C_SOREG, 14, 0, 0 }, - { ABL, C_NONE, C_NONE, C_REG, 51, 4, 0 }, - { ABX, C_NONE, C_NONE, C_REG, 15, 8, 0 }, - { ABX, C_NONE, C_NONE, C_HREG, 15, 8, 0 }, - { ABXRET, C_NONE, C_NONE, C_REG, 45, 2, 0 }, - { ABXRET, C_NONE, C_NONE, C_HREG, 45, 2, 0 }, - { ASWI, C_NONE, C_NONE, C_LCON, 16, 2, 0 }, - { AWORD, C_NONE, C_NONE, C_LCON, 17, 4, 0 }, - { AWORD, C_NONE, C_NONE, C_GCON, 17, 4, 0 }, - { AWORD, C_NONE, C_NONE, C_ADDR, 17, 4, 0 }, - { ADWORD, C_LCON, C_NONE, C_LCON, 50, 8, 0 }, - { AMOVW, C_SAUTO, C_NONE, C_REG, 18, 2, REGSP }, - { AMOVW, C_LAUTO, C_NONE, C_REG, 33, 6, 0, LFROM }, - // { AMOVW, C_OFFPC, C_NONE, C_REG, 18, 2, REGPC, LFROM }, - { AMOVW, C_SOREG, C_NONE, C_REG, 19, 2, 0 }, - { AMOVHU, C_SOREG, C_NONE, C_REG, 19, 2, 0 }, - { AMOVBU, C_SOREG, C_NONE, C_REG, 19, 2, 0 }, - { AMOVW, C_REG, C_NONE, C_SAUTO, 20, 2, 0 }, - { AMOVW, C_REG, C_NONE, C_LAUTO, 34, 6, 0, LTO }, - { AMOVW, C_REG, C_NONE, C_SOREG, 21, 2, 0 }, - { AMOVH, C_REG, C_NONE, C_SOREG, 21, 2, 0 }, - { AMOVB, C_REG, C_NONE, C_SOREG, 21, 2, 0 }, - { AMOVHU, C_REG, C_NONE, C_SOREG, 21, 2, 0 }, - { AMOVBU, C_REG, C_NONE, C_SOREG, 21, 2, 0 }, - { AMOVW, C_REG, C_NONE, C_REG, 22, 2, 0 }, - { AMOVB, C_REG, C_NONE, C_REG, 23, 4, 0 }, - { AMOVH, C_REG, C_NONE, C_REG, 23, 4, 0 }, - { AMOVBU, C_REG, C_NONE, C_REG, 23, 4, 0 }, - { AMOVHU, C_REG, C_NONE, C_REG, 23, 4, 0 }, - { AMOVH, C_SOREG, C_NONE, C_REG, 24, 4, 0 }, - { AMOVB, C_SOREG, C_NONE, C_REG, 24, 4, 0 }, - { AMOVW, C_SACON, C_NONE, C_REG, 25, 2, 0 }, - { AMOVW, C_LACON, C_NONE, C_REG, 35, 4, 0 }, - { AMOVW, C_GACON, C_NONE, C_REG, 35, 4, 0, LFROM }, - { AMOVM, C_LCON, C_NONE, C_REG, 26, 2, 0 }, - { AMOVM, C_REG, C_NONE, C_LCON, 27, 2, 0 }, - { AMOVW, C_LOREG, C_NONE, C_REG, 28, 4, 0 }, - { AMOVH, C_LOREG, C_NONE, C_REG, 28, 4, 0 }, - { AMOVB, C_LOREG, C_NONE, C_REG, 28, 4, 0 }, - { AMOVHU, C_LOREG, C_NONE, C_REG, 28, 4, 0 }, - { AMOVBU, C_LOREG, C_NONE, C_REG, 28, 4, 0 }, - { AMOVW, C_REG, C_NONE, C_LOREG, 29, 4, 0 }, - { AMOVH, C_REG, C_NONE, C_LOREG, 29, 4, 0 }, - { AMOVB, C_REG, C_NONE, C_LOREG, 29, 4, 0 }, - { AMOVHU, C_REG, C_NONE, C_LOREG, 29, 4, 0 }, - { AMOVBU, C_REG, C_NONE, C_LOREG, 29, 4, 0 }, - { AMOVW, C_GOREG, C_NONE, C_REG, 28, 4, 0, LFROM }, - { AMOVH, C_GOREG, C_NONE, C_REG, 28, 4, 0, LFROM }, - { AMOVB, C_GOREG, C_NONE, C_REG, 28, 4, 0, LFROM }, - { AMOVHU, C_GOREG, C_NONE, C_REG, 28, 4, 0, LFROM }, - { AMOVBU, C_GOREG, C_NONE, C_REG, 28, 4, 0, LFROM }, - { AMOVW, C_REG, C_NONE, C_GOREG, 29, 4, 0, LTO }, - { AMOVH, C_REG, C_NONE, C_GOREG, 29, 4, 0, LTO }, - { AMOVB, C_REG, C_NONE, C_GOREG, 29, 4, 0, LTO }, - { AMOVHU, C_REG, C_NONE, C_GOREG, 29, 4, 0, LTO }, - { AMOVBU, C_REG, C_NONE, C_GOREG, 29, 4, 0, LTO }, - { AMOVW, C_ADDR, C_NONE, C_REG, 30, 4, 0, LFROM }, - { AMOVH, C_ADDR, C_NONE, C_REG, 32, 6, 0, LFROM }, - { AMOVB, C_ADDR, C_NONE, C_REG, 32, 6, 0, LFROM }, - { AMOVHU, C_ADDR, C_NONE, C_REG, 30, 4, 0, LFROM }, - { AMOVBU, C_ADDR, C_NONE, C_REG, 30, 4, 0, LFROM }, - { AMOVW, C_REG, C_NONE, C_ADDR, 31, 4, 0, LTO }, - { AMOVH, C_REG, C_NONE, C_ADDR, 31, 4, 0, LTO }, - { AMOVB, C_REG, C_NONE, C_ADDR, 31, 4, 0, LTO }, - { AMOVHU, C_REG, C_NONE, C_ADDR, 31, 4, 0, LTO }, - { AMOVBU, C_REG, C_NONE, C_ADDR, 31, 4, 0, LTO }, - - { AXXX, C_NONE, C_NONE, C_NONE, 0, 2, 0 }, -}; - -#define OPCNTSZ 52 -int opcount[OPCNTSZ]; - -// is this too pessimistic ? -int -brextra(Prog *p) -{ - int c; - - // +2 is for padding - if(p->as == ATEXT) - return 0-0+2; - if(!isbranch(p)) - diag("bad op in brextra()"); - c = thumbaclass(&p->to, p); - switch(p->as){ - case AB: - if(c != C_SBRA) - return 0; - return 8-2+2; - case ABL: - if(c != C_SBRA) - return 0; - return 14-4+2; - case ABX: - if(c == C_REG || c == C_HREG) - return 0; -#ifdef CALLEEBX - diag("ABX $I in brextra"); -#endif - if(c != C_SBRA) - return 0; - return 14-10+2; - default: - if(c == C_GBRA) - return 0; - if(c == C_LBRA) - return 10-4+2; - return 10-2+2; - } - return 0; -} - -#define high(r) ((r)>=8) - -static int32 -mv(Prog *p, int r, int off) -{ - int v, o; - if(p != nil && p->cond != nil){ // in literal pool - v = p->cond->pc - p->pc - 4; - if(p->cond->pc & 3) - diag("mv: bad literal pool alignment"); - if(v & 3) - v += 2; // ensure M(4) offset - mult(p, v, 4); - off = v/4; - numr(p, off, 0, 255); - o = 0x9<<11; - } - else{ - numr(p, off, 0, 255); - o = 0x4<<11; - } - o |= (r<<8) | off; - return o; -} - -static void -mvcon(Prog *p, int r, int c, int32 *o1, int32 *o2) -{ - int op = 0, n = 0; - - if(c >= 0 && c <= 255) - diag("bad c in mvcon"); - if(c >= -255 && c < 0) // mv, neg - c = -c; - else if(c >= 256 && c <= 510){ // mv, add - n = rand()%(511-c) + (c-255); - c -= n; - // n = c-255; - // c = 255; - op = AADD; - } - else{ - if(c < 0) - diag("-ve in mvcon"); - while(!(c & 1)){ - n++; - c >>= 1; - } - if(c >= 0 && c <= 255) // mv, lsl - op = ASLL; - else - diag("bad shift in mvcon"); - } - *o1 = mv(p, r, c); - switch(op){ - case 0: - *o2 = (1<<14) | (9<<6) | (r<<3) | r; - break; - case AADD: - *o2 = (6<<11) | (r<<8) | n; - break; - case ASLL: - *o2 = (n<<6) | (r<<3) | r; - break; - } -} - -static int32 -mvlh(int rs, int rd) -{ - int o = 0x46<<8; - - if(high(rs)){ - rs -= 8; - o |= 1<<6; - } - if(high(rd)){ - rd -= 8; - o |= 1<<7; - } - o |= (rs<<3) | rd; - return o; -} - -void -thumbbuildop() -{ - int i, n, r; - Optab *optab = thumboptab; - Oprang *oprange = thumboprange; - - for(n=0; optab[n].as != AXXX; n++) - ; - qsort(optab, n, sizeof(optab[0]), ocmp); - for(i=0; i<n; i++) { - r = optab[i].as; - oprange[r].start = optab+i; - while(optab[i].as == r) - i++; - oprange[r].stop = optab+i; - i--; - - switch(r) - { - default: - break; - case ABEQ: - oprange[ABNE] = oprange[r]; - oprange[ABCS] = oprange[r]; - oprange[ABHS] = oprange[r]; - oprange[ABCC] = oprange[r]; - oprange[ABLO] = oprange[r]; - oprange[ABMI] = oprange[r]; - oprange[ABPL] = oprange[r]; - oprange[ABVS] = oprange[r]; - oprange[ABVC] = oprange[r]; - oprange[ABHI] = oprange[r]; - oprange[ABLS] = oprange[r]; - oprange[ABGE] = oprange[r]; - oprange[ABLT] = oprange[r]; - oprange[ABGT] = oprange[r]; - oprange[ABLE] = oprange[r]; - break; - case AMVN: - oprange[AADC] = oprange[r]; - oprange[ASBC] = oprange[r]; - oprange[AMUL] = oprange[r]; - oprange[AAND] = oprange[r]; - oprange[AEOR] = oprange[r]; - oprange[AORR] = oprange[r]; - oprange[ABIC] = oprange[r]; - oprange[AMULU] = oprange[r]; - break; - case ACMN: - oprange[ATST] = oprange[r]; - break; - case ASRL: - oprange[ASRA] = oprange[r]; - oprange[ASLL] = oprange[r]; - break; - case AADD: - oprange[ASUB] = oprange[r]; - break; - } - } -} - -void -thumbasmout(Prog *p, Optab *o) -{ - int32 o1, o2, o3, o4, o5, o6, o7, v; - int r, rf, rt; - - rf = p->from.reg; - rt = p->to.reg; - r = p->reg; - o1 = o2 = o3 = o4 = o5 = o6 = o7 = 0; -if(debug['P']) print("%ux: %P type %d %d\n", (uint32)(p->pc), p, o->type, p->align); - opcount[o->type] += o->size; - switch(o->type) { - default: - diag("unknown asm %d", o->type); - prasm(p); - break; - case 0: /* pseudo ops */ -if(debug['G']) print("%ux: %s: thumb\n", (uint32)(p->pc), p->from.sym->name); - break; - case 1: /* op R, -, R or op R, R, - */ - o1 = thumboprr(p->as); - if(rt == NREG) - rt = r; - lowreg(p, rf); - lowreg(p, rt); - o1 |= (0x10<<10) | (rf<<3) | rt; - break; - case 2: /* add/sub R, R, R or add/sub R, -, R */ - o1 = p->as == AADD ? 0x0<<9 : 0x1<<9; - if(r == NREG) - r = rt; - lowreg(p, rf); - lowreg(p, r); - lowreg(p, rt); - o1 |= (0x6<<10) | (rf<<6) | (r<<3) | rt; - break; - case 3: /* add/sub $I, R, R or add/sub $I, -, R */ - thumbaclass(&p->from, p); - o1 = p->as == AADD ? 0x0<<9 : 0x1<<9; - if(r == NREG) - r = rt; - numr(p, instoffset, 0, 7); - lowreg(p, r); - lowreg(p, rt); - o1 |= (0x7<<10) | (instoffset<<6) | (r<<3) | rt; - break; - case 4: /* shift $I, R, R or shift $I, -, R */ - thumbaclass(&p->from, p); - if(instoffset < 0) - diag("negative shift in thumbasmout"); - instoffset %= 32; - o1 = thumbopri(p->as); - if(r == NREG) - r = rt; - numr(p, instoffset, 0, 31); - lowreg(p, r); - lowreg(p, rt); - o1 |= (0x0<<13) | (instoffset<<6) | (r<<3) | rt; - break; - case 5: /* add/sub/mov $I, -, R or cmp $I, R, - */ - thumbaclass(&p->from, p); - o1 = thumbopri(p->as); - if(rt == NREG) - rt = r; - numr(p, instoffset, 0, 255); - lowreg(p, rt); - o1 |= (0x1<<13) | (rt<<8) | instoffset; - break; - case 6: /* add $I, PC/SP, R */ - if(p->as == ASUB) - diag("subtract in add $I, PC/SP, R"); - thumbaclass(&p->from, p); - o1 = r == REGSP ? 0x1<<11 : 0x0<<11; - numr(p, instoffset, 0, 255); - regis(p, r, REGSP, REGPC); - lowreg(p, rt); - o1 |= (0xa<<12) | (rt<<8) | instoffset; - break; - case 7: /* add, sub $I, SP */ - thumbaclass(&p->from, p); - o1 = p->as == AADD ? 0x0<<7 : 0x1<<7; - numr(p, instoffset, 0, 508); - mult(p, instoffset, 4); - regis(p, rt, REGSP, REGSP); - o1 |= (0xb0<<8) | (instoffset>>2); - break; - case 8: /* add/mov/cmp R, R where at least 1 reg is high */ - o1 = 0; - if(rt == NREG) - rt = r; - if(high(rf)){ - o1 |= 1<<6; - rf -= 8; - } - if(high(rt)){ - o1 |= 2<<6; - rt -= 8; - } - if(o1 == 0) - diag("no high register(%P)", p); - o1 |= thumbophh(p->as); - o1 |= (0x11<<10) | (rf<<3) | rt; - break; - case 9: /* B $I */ - thumbaclass(&p->to, p); - numr(p, instoffset, -2048, 2046); - o1 = (0x1c<<11) | ((instoffset>>1)&0x7ff); - break; - case 10: /* Bcc $I */ - thumbaclass(&p->to, p); - numr(p, instoffset, -256, 254); - o1 = thumbopbra(p->as); - o1 |= (0xd<<12) | ((instoffset>>1)&0xff); - break; - case 11: /* BL $I */ - thumbaclass(&p->to, p); - numr(p, instoffset, -4194304, 4194302); - o1 = (0x1e<<11) | ((instoffset>>12)&0x7ff); - o2 = (0x1f<<11) | ((instoffset>>1)&0x7ff); - break; - case 12: /* BX $I */ -#ifdef CALLEEBX - diag("BX $I case"); -#endif - thumbaclass(&p->to, p); - if(p->to.sym->thumb) - instoffset |= 1; // T bit - o1 = mvlh(REGPC, REGTMPT); - o2 = (0x6<<11) | (REGTMPT<<8) | 7; // add 7, RTMP (T bit + PC offset) - o3 = mvlh(REGTMPT, REGLINK); - o4 = mv(nil, REGTMPT, instoffset); - o5 = (0x11c<<6) | (REGTMPT<<3); - // o1 = mv(nil, REGTMPT, v); - // o2 = (0x11b<<6) | (REGPC<<3) | REGLINK; - // o3 = (0x11c<<6) | (REGTMPT<<3); - break; - case 13: /* B O(R) */ - diag("B O(R)"); - break; - case 14: /* BL O(R) */ - diag("BL O(R)"); - break; - case 15: /* BX R */ - o1 = mvlh(REGPC, REGTMPT); - o2 = (0x6<<11) | (REGTMPT<<8) | 5; // add 5, RTMP (T bit + PC offset) - o3 = mvlh(REGTMPT, REGLINK); - o4 = 0; - if(high(rt)){ - rt -= 8; - o4 |= 1<<6; - } - o4 |= (0x8e<<7) | (rt<<3); - // o1 = (0x11c<<6) | (rt<<3); - break; - case 16: /* SWI $I */ - thumbaclass(&p->to, p); - numr(p, instoffset, 0, 255); - o1 = (0xdf<<8) | instoffset; - break; - case 17: /* AWORD */ - thumbaclass(&p->to, p); - o1 = instoffset&0xffff; - o2 = (instoffset>>16)&0xffff; - break; - case 18: /* AMOVW O(SP), R and AMOVW O(PC), R */ - thumbaclass(&p->from, p); - rf = o->param; - o1 = rf == REGSP ? 0x13<<11 : 0x9<<11; - regis(p, rf, REGSP, REGPC); - lowreg(p, rt); - mult(p, instoffset, 4); - numr(p, instoffset/4, 0, 255); - o1 |= (rt<<8) | (instoffset/4); - break; - case 19: /* AMOVW... O(R), R */ - thumbaclass(&p->from, p); - o1 = thumbopmv(p->as, 1); - v = 4; - if(p->as == AMOVHU) - v = 2; - else if(p->as == AMOVBU) - v = 1; - mult(p, instoffset, v); - lowreg(p, rf); - lowreg(p, rt); - numr(p, instoffset/v, 0, 31); - o1 |= ((instoffset/v)<<6) | (rf<<3) | rt; - break; - case 20: /* AMOVW R, O(SP) */ - thumbaclass(&p->to, p); - o1 = 0x12<<11; - if(rt != NREG) regis(p, rt, REGSP, REGSP); - lowreg(p, rf); - mult(p, instoffset, 4); - numr(p, instoffset/4, 0, 255); - o1 |= (rf<<8) | (instoffset/4); - break; - case 21: /* AMOVW... R, O(R) */ - thumbaclass(&p->to, p); - o1 = thumbopmv(p->as, 0); - v = 4; - if(p->as == AMOVHU || p->as == AMOVH) - v = 2; - else if(p->as == AMOVBU || p->as == AMOVB) - v = 1; - lowreg(p, rf); - lowreg(p, rt); - mult(p, instoffset, v); - numr(p, instoffset/v, 0, 31); - o1 |= ((instoffset/v)<<6) | (rt<<3) | rf; - break; - case 22: /* AMOVW R, R -> ASLL $0, R, R */ - o1 = thumbopri(ASLL); - lowreg(p, rf); - lowreg(p, rt); - o1 |= (0x0<<13) | (rf<<3) | rt; - break; - case 23: /* AMOVB/AMOVH/AMOVBU/AMOVHU R, R */ - o1 = thumbopri(ASLL); - o2 = p->as == AMOVB || p->as == AMOVH ? thumbopri(ASRA) : thumbopri(ASRL); - v = p->as == AMOVB || p->as == AMOVBU ? 24 : 16; - lowreg(p, rf); - lowreg(p, rt); - o1 |= (0x0<<13) | (v<<6) | (rf<<3) | rt; - o2 |= (0x0<<13) | (v<<6) | (rt<<3) | rt; - break; - case 24: /* AMOVH/AMOVB O(R), R -> AMOVH/AMOVB [R, R], R */ - thumbaclass(&p->from, p); - lowreg(p, rf); - lowreg(p, rt); - if(rf == rt) - r = REGTMPT; - else - r = rt; - if(p->as == AMOVB) - numr(p, instoffset, 0, 31); - else{ - mult(p, instoffset, 2); - numr(p, instoffset, 0, 62); - } - o1 = mv(p, r, instoffset); - o2 = p->as == AMOVH ? 0x2f<<9 : 0x2b<<9; - o2 |= (r<<6) | (rf<<3) | rt; - break; - case 25: /* MOVW $sacon, R */ - thumbaclass(&p->from, p); -// print("25: %d %d %d %d\n", instoffset, rf, r, rt); - if(rf == NREG) - rf = REGSP; - lowreg(p, rt); - if(rf == REGSP){ - mult(p, instoffset, 4); - numr(p, instoffset>>2, 0, 255); - o1 = (0x15<<11) | (rt<<8) | (instoffset>>2); // add $O, SP, R - } - else if(rf == rt){ - numr(p, instoffset, 0, 255); - o1 = (0x6<<11) | (rt<<8) | instoffset; // add $O, R - } - else{ - lowreg(p, rf); - numr(p, instoffset, 0, 7); - o1 = (0xe<<9) | (instoffset<<6) | (rf<<3) | rt; // add $O, Rs, Rd - } - break; - case 26: /* AMOVM $c, oreg -> stmia */ - lowreg(p, rt); - numr(p, p->from.offset, -256, 255); - o1 = (0x18<<11) | (rt<<8) | (p->from.offset&0xff); - break; - case 27: /* AMOVM oreg, $c ->ldmia */ - lowreg(p, rf); - numr(p, p->to.offset, -256, 256); - o1 = (0x19<<11) | (rf<<8) | (p->to.offset&0xff); - break; - case 28: /* AMOV* O(R), R -> AMOV* [R, R], R (offset large) */ - thumbaclass(&p->from, p); - lowreg(p, rf); - lowreg(p, rt); - if(rf == rt) - r = REGTMPT; - else - r = rt; - o1 = mv(p, r, instoffset); - o2 = thumboprrr(p->as, 1); - o2 |= (r<<6) | (rf<<3) | rt; - break; - case 29: /* AMOV* R, O(R) -> AMOV* R, [R, R] (offset large) */ - thumbaclass(&p->to, p); - lowreg(p, rf); - lowreg(p, rt); - if(rt == REGTMPT){ // used as tmp reg - if(instoffset >= 0 && instoffset <= 255){ - o1 = (1<<13) | (2<<11) | (rt<<8) | instoffset; // add $O, R7 - o2 = thumbopirr(p->as, 0); - o2 |= (0<<6) | (rt<<3) | rf; // mov* R, 0(R) - } - else - diag("big offset - case 29"); - } - else{ - o1 = mv(p, REGTMPT, instoffset); - o2 = thumboprrr(p->as, 0); - o2 |= (REGTMPT<<6) | (rt<<3) | rf; - } - break; - case 30: /* AMOVW... *addr, R */ - diag("likely broken"); // does this still refer to SB? - thumbaclass(&p->from, p); - o1 = mv(p, rt, instoffset); // MOV addr, rtmp - o2 = thumbopmv(p->as, 1); - lowreg(p, rt); - o2 |= (rt<<3) | rt; // MOV* 0(rtmp), R - break; - case 31: /* AMOVW... R, *addr */ - diag("likely broken"); // does this still refer to SB? - thumbaclass(&p->to, p); - o1 = mv(p, REGTMPT, instoffset); - o2 = thumbopmv(p->as, 0); - lowreg(p, rf); - o2 |= (REGTMPT<<3) | rf; - break; - case 32: /* AMOVH/AMOVB *addr, R -> AMOVH/AMOVB [R, R], R */ - thumbaclass(&p->from, p); - o1 = mv(p, rt, instoffset); - lowreg(p, rt); - o2 = mv(nil, REGTMPT, 0); - o3 = p->as == AMOVH ? 0x2f<<9 : 0x2b<<9; - o3 |= (REGTMPT<<6) | (rt<<3) | rt; - break; - case 33: /* AMOVW O(SP), R (O large) */ - thumbaclass(&p->from, p); - lowreg(p, rt); - o1 = mv(p, rt, instoffset); - o2 = (0x111<<6) | (REGSP-8)<<3 | rt; // add SP, rt - o3 = thumbopmv(p->as, 1); - o3 |= (rt<<3) | rt; - break; - case 34: /* AMOVW R, O(SP) (O large) */ - thumbaclass(&p->to, p); - lowreg(p, rf); - o1 = mv(p, REGTMPT, instoffset); - o2 = (0x111<<6) | (REGSP-8)<<3 | REGTMPT; // add SP, REGTMP - o3 = thumbopmv(p->as, 0); - o3 |= (REGTMPT<<3) | rf; - break; - case 35: /* AMOVW $lacon, R */ - thumbaclass(&p->from, p); - lowreg(p, rt); - if(rf == NREG) - rf = REGSP; - if(rf == rt) - rf = r = REGTMPT; - else - r = rt; -// print("35: io=%d rf=%d rt=%d\n", instoffset, rf, rt); - o1 = mv(p, r, instoffset); // mov O, Rd - if(high(rf)) - o2 = (0x44<<8) | (0x1<<6) | ((rf-8)<<3) | rt; // add Rs, Rd - else - o2 = (0x6<<10) | (rf<<6) | (rt<<3) | rt; // add Rs, Rd - break; - case 36: /* AADD/ASUB $i, r, r when $i too big */ - thumbaclass(&p->from, p); - lowreg(p, r); - lowreg(p, rt); - o1 = mv(p, REGTMPT, instoffset); - o2 = p->as == AADD ? 0xc<<9 : 0xd<<9; - o2 |= (REGTMPT<<6) | (r<<3) | rt; - break; - case 37: /* AADD/ASUB $i, r when $i too big */ - thumbaclass(&p->from, p); - lowreg(p, rt); - o1 = mv(p, REGTMPT, instoffset); - o2 = p->as == AADD ? 0xc<<9 : 0xd<<9; - o2 |= (REGTMPT<<6) | (rt<<3) | rt; - break; - case 38: /* AMOVW $i, r when $i too big */ - thumbaclass(&p->from, p); - lowreg(p, rt); - o1 = mv(p, rt, instoffset); - break; - case 39: /* ACMP $i, r when $i too big */ - thumbaclass(&p->from, p); - lowreg(p, r); - o1 = mv(p, REGTMPT, instoffset); - o2 = (0x10a<<6) | (REGTMPT<<3) | r; - break; - case 40: /* add, sub $I, SP when $I large*/ - thumbaclass(&p->from, p); - if(p->as == ASUB) - instoffset = -instoffset; - o1 = mv(p, REGTMPT, instoffset); - o2 = (0x112<<6) | (REGTMPT<<3) | (REGSP-8); - regis(p, rt, REGSP, REGSP); - break; - case 41: /* BL LBRA */ - thumbaclass(&p->to, p); - o1 = (0x9<<11) | (REGTMPT<<8); // mov 0(pc), r7 - o2 = mvlh(REGTMPT, REGPC); // mov r7, pc - o3 = instoffset&0xffff; // $lab - o4 = (instoffset>>16)&0xffff; - break; - case 42: /* Bcc GBRA */ - thumbaclass(&p->to, p); - o1 = (0xd<<12) | thumbopbra(relinv(p->as)) | (6>>1); // bccnot - // ab lbra - o2 = (0x9<<11) | (REGTMPT<<8); // mov 0(pc), r7 - o3 = mvlh(REGTMPT, REGPC); // mov r7, pc - o4 = instoffset&0xffff; // $lab - o5 = (instoffset>>16)&0xffff; - break; - case 43: /* BL LBRA */ - thumbaclass(&p->to, p); - o1 = mvlh(REGPC, REGTMPT); // mov pc, r7 - o2 = (0x6<<11) | (REGTMPT<<8) | 10; // add 10, r7 - o3 = mvlh(REGTMPT, REGLINK); // mov r7, lr - o4 = (0x9<<11) | (REGTMPT<<8); // mov o(pc), r7 - o5 = mvlh(REGTMPT, REGPC); // mov r7, pc - o6 = instoffset&0xffff; // $lab - o7 = (instoffset>>16)&0xffff; - break; - case 44: /* BX LBRA */ -#ifdef CALLEEBX - diag("BX LBRA case"); -#endif - thumbaclass(&p->to, p); - if(p->to.sym->thumb) - instoffset |= 1; // T bit - o1 = mvlh(REGPC, REGTMPT); // mov pc, r7 - o2 = (0x6<<11) | (REGTMPT<<8) | 11; // add 11, r7 - o3 = mvlh(REGTMPT, REGLINK); // mov r7, lr - o4 = (0x9<<11) | (REGTMPT<<8); // mov o(pc), r7 - o5 = (0x11c<<6) | (REGTMPT<<3); // bx r7 - o6 = instoffset&0xffff; // $lab - o7 = (instoffset>>16)&0xffff; - break; - case 45: /* BX R when returning from fn */ - o1 = 0; - if(high(rt)){ - rt -= 8; - o1 |= 1<<6; - } - o1 |= (0x8e<<7) | (rt<<3); - break; - case 46: /* Bcc LBRA */ - thumbaclass(&p->to, p); - o1 = (0xd<<12) | thumbopbra(relinv(p->as)) | (0>>1); // bccnot - // ab lbra - instoffset -= 2; - numr(p, instoffset, -2048, 2046); - o2 = (0x1c<<11) | ((instoffset>>1)&0x7ff); - break; - case 47: /* mov $i, R where $i can be built */ - thumbaclass(&p->from, p); - mvcon(p, rt, instoffset, &o1, &o2); - break; - case 48: /* ACMP $i, r when $i built up */ - thumbaclass(&p->from, p); - lowreg(p, r); - mvcon(p, REGTMPT, instoffset, &o1, &o2); - o3 = (0x10a<<6) | (REGTMPT<<3) | r; - break; - case 49: /* AADD $i, r, r when $i is between 0 and 255 - could merge with case 36 */ - thumbaclass(&p->from, p); - lowreg(p, r); - lowreg(p, rt); - numr(p, instoffset, 0, 255); - o1 = mv(p, REGTMPT, instoffset); - o2 = p->as == AADD ? 0xc<<9 : 0xd<<9; - o2 |= (REGTMPT<<6) | (r<<3) | rt; - break; - case 50: /* ADWORD */ - thumbaclass(&p->from, p); - o1 = instoffset&0xffff; - o2 = (instoffset>>16)&0xffff; - thumbaclass(&p->to, p); - o3 = instoffset&0xffff; - o4 = (instoffset>>16)&0xffff; - break; - case 51: /* BL r */ - o1 = mvlh(REGPC, REGLINK); // mov pc, lr - o2 = mvlh(rt, REGPC); // mov r, pc - break; - } - - v = p->pc; - switch(o->size) { - default: - if(debug['a']) - Bprint(&bso, " %.8ux:\t\t%P\n", v, p); - break; - case 2: - if(debug['a']) - Bprint(&bso, " %.8ux: %.8ux\t%P\n", v, o1, p); - hputl(o1); - break; - case 4: - if(debug['a']) - Bprint(&bso, " %.8ux: %.8ux %.8ux\t%P\n", v, o1, o2, p); - hputl(o1); - hputl(o2); - break; - case 6: - if(debug['a']) - Bprint(&bso, "%.8ux: %.8ux %.8ux %.8ux\t%P\n", v, o1, o2, o3, p); - hputl(o1); - hputl(o2); - hputl(o3); - break; - case 8: - if(debug['a']) - Bprint(&bso, "%.8ux: %.8ux %.8ux %.8ux %.8ux\t%P\n", v, o1, o2, o3, o4, p); - hputl(o1); - hputl(o2); - hputl(o3); - hputl(o4); - break; - case 10: - if(debug['a']) - Bprint(&bso, "%.8ux: %.8ux %.8ux %.8ux %.8ux %.8ux\t%P\n", v, o1, o2, o3, o4, o5, p); - hputl(o1); - hputl(o2); - hputl(o3); - hputl(o4); - hputl(o5); - break; - case 12: - if(debug['a']) - Bprint(&bso, "%.8ux: %.8ux %.8ux %.8ux %.8ux %.8ux %.8ux\t%P\n", v, o1, o2, o3, o4, o5, o6, p); - hputl(o1); - hputl(o2); - hputl(o3); - hputl(o4); - hputl(o5); - hputl(o6); - break; - case 14: - if(debug['a']) - Bprint(&bso, "%.8ux: %.8ux %.8ux %.8ux %.8ux %.8ux %.8ux %.8ux\t%P\n", v, o1, o2, o3, o4, o5, o6, o7, p); - hputl(o1); - hputl(o2); - hputl(o3); - hputl(o4); - hputl(o5); - hputl(o6); - hputl(o7); - break; - } - if(debug['G']){ - if(o->type == 17){ - print("%x: word %d\n", p->pc, (o2<<16)+o1); - return; - } - if(o->type == 50){ - print("%x: word %d\n", p->pc, (o2<<16)+o1); - print("%x: word %d\n", p->pc, (o4<<16)+o3); - return; - } - if(o->size > 0) dis(o1, p->pc); - if(o->size > 2) dis(o2, p->pc+2); - if(o->size > 4) dis(o3, p->pc+4); - if(o->size > 6) dis(o4, p->pc+6); - if(o->size > 8) dis(o5, p->pc+8); - if(o->size > 10) dis(o6, p->pc+10); - if(o->size > 12) dis(o7, p->pc+12); - // if(o->size > 14) dis(o8, p->pc+14); - } -} - -static int32 -thumboprr(int a) -{ - switch(a) { - case AMVN: return 0xf<<6; - case ACMP: return 0xa<<6; - case ACMN: return 0xb<<6; - case ATST: return 0x8<<6; - case AADC: return 0x5<<6; - case ASBC: return 0x6<<6; - case AMUL: - case AMULU: return 0xd<<6; - case AAND: return 0x0<<6; - case AEOR: return 0x1<<6; - case AORR: return 0xc<<6; - case ABIC: return 0xe<<6; - case ASRL: return 0x3<<6; - case ASRA: return 0x4<<6; - case ASLL: return 0x2<<6; - } - diag("bad thumbop oprr %d", a); - prasm(curp); - return 0; -} - -static int32 -thumbopirr(int a, int ld) -{ - if(ld) - diag("load in thumbopirr"); - switch(a){ - case AMOVW: return 0xc<<11; - case AMOVH: - case AMOVHU: return 0x10<<11; - case AMOVB: - case AMOVBU: return 0xe<<11; - } - return 0; -} - -static int32 -thumboprrr(int a, int ld) -{ - if(ld){ - switch(a){ - case AMOVW: return 0x2c<<9; - case AMOVH: return 0x2f<<9; - case AMOVB: return 0x2b<<9; - case AMOVHU: return 0x2d<<9; - case AMOVBU: return 0x2e<<9; - } - } - else{ - switch(a){ - case AMOVW: return 0x28<<9; - case AMOVHU: - case AMOVH: return 0x29<<9; - case AMOVBU: - case AMOVB: return 0x2a<<9; - } - } - diag("bad thumbop oprrr %d", a); - prasm(curp); - return 0; -} - -static int32 -thumbopri(int a) -{ - switch(a) { - case ASRL: return 0x1<<11; - case ASRA: return 0x2<<11; - case ASLL: return 0x0<<11; - case AADD: return 0x2<<11; - case ASUB: return 0x3<<11; - case AMOVW: return 0x0<<11; - case ACMP: return 0x1<<11; - } - diag("bad thumbop opri %d", a); - prasm(curp); - return 0; -} - -static int32 -thumbophh(int a) -{ - switch(a) { - case AADD: return 0x0<<8; - case AMOVW: return 0x2<<8; - case ACMP: return 0x1<<8; - } - diag("bad thumbop ophh %d", a); - prasm(curp); - return 0; -} - -static int32 -thumbopbra(int a) -{ - switch(a) { - case ABEQ: return 0x0<<8; - case ABNE: return 0x1<<8; - case ABCS: return 0x2<<8; - case ABHS: return 0x2<<8; - case ABCC: return 0x3<<8; - case ABLO: return 0x3<<8; - case ABMI: return 0x4<<8; - case ABPL: return 0x5<<8; - case ABVS: return 0x6<<8; - case ABVC: return 0x7<<8; - case ABHI: return 0x8<<8; - case ABLS: return 0x9<<8; - case ABGE: return 0xa<<8; - case ABLT: return 0xb<<8; - case ABGT: return 0xc<<8; - case ABLE: return 0xd<<8; - } - diag("bad thumbop opbra %d", a); - prasm(curp); - return 0; -} - -static int32 -thumbopmv(int a, int ld) -{ - switch(a) { - case AMOVW: return (ld ? 0xd : 0xc)<<11; - case AMOVH: - case AMOVHU: return (ld ? 0x11: 0x10)<<11; - case AMOVB: - case AMOVBU: return (ld ? 0xf : 0xe)<<11; - } - diag("bad thumbop opmv %d", a); - prasm(curp); - return 0; -} - -static void -lowreg(Prog *p, int r) -{ - if(high(r)) - diag("high reg [%P]", p); -} - -static void -mult(Prog *p, int n, int m) -{ - if(m*(n/m) != n) - diag("%d not M(%d) [%P]", n, m, p); -} - -static void -numr(Prog *p, int n, int min, int max) -{ - if(n < min || n > max) - diag("%d not in %d-%d [%P]", n, min, max, p); -} - -static void -regis(Prog *p, int r, int r1, int r2) -{ - if(r != r1 && r != r2) - diag("reg %d not %d or %d [%P]", r, r1, r2, p); -} - -void -hputl(int n) -{ - cbp[1] = n>>8; - cbp[0] = n; - cbp += 2; - cbc -= 2; - if(cbc <= 0) - cflush(); -} - -void -thumbcount() -{ - int i, c = 0, t = 0; - - for (i = 0; i < OPCNTSZ; i++) - t += opcount[i]; - if(t == 0) - return; - for (i = 0; i < OPCNTSZ; i++){ - c += opcount[i]; - print("%d: %d %d %d%%\n", i, opcount[i], c, (opcount[i]*100+t/2)/t); - } -} - -char *op1[] = { "lsl", "lsr", "asr" }; -char *op2[] = { "add", "sub" }; -char *op3[] = { "movw", "cmp", "add", "sub" }; -char *op4[] = { "and", "eor", "lsl", "lsr", "asr", "adc", "sbc", "ror", - "tst", "neg", "cmp", "cmpn", "or", "mul", "bitc", "movn" }; -char *op5[] = { "add", "cmp", "movw", "bx" }; -char *op6[] = { "smovw", "smovh", "smovb", "lmovb", "lmovw", "lmovhu", "lmovbu", "lmovh" }; -char *op7[] = { "smovw", "lmovw", "smovb", "lmovbu" }; -char *op8[] = { "smovh", "lmovhu" }; -char *op9[] = { "smovw", "lmovw" }; -char *op10[] = { "push", "pop" }; -char *op11[] = { "stmia", "ldmia" }; - -char *cond[] = { "eq", "ne", "hs", "lo", "mi", "pl", "vs", "vc", - "hi", "ls", "ge", "lt", "gt", "le", "al", "nv" }; - -#define B(h, l) bits(i, h, l) -#define IMM(h, l) B(h, l) -#define REG(h, l) reg(B(h, l)) -#define LHREG(h, l, lh) lhreg(B(h, l), B(lh, lh)) -#define COND(h, l) cond[B(h, l)] -#define OP1(h, l) op1[B(h, l)] -#define OP2(h, l) op2[B(h, l)] -#define OP3(h, l) op3[B(h, l)] -#define OP4(h, l) op4[B(h, l)] -#define OP5(h, l) op5[B(h, l)] -#define OP6(h, l) op6[B(h, l)] -#define OP7(h, l) op7[B(h, l)] -#define OP8(h, l) op8[B(h, l)] -#define OP9(h, l) op9[B(h, l)] -#define OP10(h, l) op10[B(h, l)] -#define OP11(h, l) op11[B(h, l)] -#define SBZ(h, l) if(IMM(h, l) != 0) diag("%x: %x bits %d,%d not zero", pc, i, h, l) -#define SNBZ(h, l) if(IMM(h, l) == 0) diag("%x: %x bits %d,%d zero", pc, i, h, l) -#define SBO(h, l) if(IMM(h, l) != 1) diag("%x: %x bits %d,%d not one", pc, i, h, l) - -static int -bits(int i, int h, int l) -{ - if(h < l) - diag("h < l in bits"); - return (i&(((1<<(h-l+1))-1)<<l))>>l; -} - -static char * -reg(int r) -{ - static char s[4][4]; - static int i = 0; - - if(r < 0 || r > 7) - diag("register %d out of range", r); - i++; - if(i == 4) - i = 0; - sprint(s[i], "r%d", r); - return s[i]; -} - -static char *regnames[] = { "sp", "lr", "pc" }; - -static char * -lhreg(int r, int lh) -{ - static char s[4][4]; - static int i = 0; - - if(lh == 0) - return reg(r); - if(r < 0 || r > 7) - diag("high register %d out of range", r); - i++; - if(i == 4) - i = 0; - if(r >= 5) - sprint(s[i], "%s", regnames[r-5]); - else - sprint(s[i], "r%d", r+8); - return s[i]; -} - -static void -illegal(int i, int pc) -{ - diag("%x: %x illegal instruction", pc, i); -} - -static void -dis(int i, int pc) -{ - static int lasto; - int o, l; - char *op; - - print("%x: %x: ", pc, i); - if(i&0xffff0000) - illegal(i, pc); - o = B(15, 13); - switch(o){ - case 0: - o = B(12, 11); - switch(o){ - case 0: - case 1: - case 2: - print("%s %d, %s, %s\n", OP1(12, 11), IMM(10, 6), REG(5, 3), REG(2, 0)); - return; - case 3: - if(B(10, 10) == 0) - print("%s %s, %s, %s\n", OP2(9, 9), REG(8, 6), REG(5, 3), REG(2, 0)); - else - print("%s %d, %s, %s\n", OP2(9, 9), IMM(8, 6), REG(5, 3), REG(2, 0)); - return; - } - case 1: - print("%s %d, %s\n", OP3(12, 11), IMM(7, 0), REG(10, 8)); - return; - case 2: - o = B(12, 10); - if(o == 0){ - print("%s %s, %s\n", OP4(9, 6), REG(5, 3), REG(2, 0)); - return; - } - if(o == 1){ - o = B(9, 8); - if(o == 3){ - SBZ(7, 7); - SBZ(2, 0); - print("%s %s\n", OP5(9, 8), LHREG(5, 3, 6)); - return; - } - SNBZ(7, 6); - print("%s %s, %s\n", OP5(9, 8), LHREG(5, 3, 6), LHREG(2, 0, 7)); - return; - } - if(o == 2 || o == 3){ - print("movw %d(pc)[%x], %s\n", 4*IMM(7, 0), 4*IMM(7, 0)+pc+4, REG(10, 8)); - return; - } - op = OP6(11, 9); - if(*op == 'l') - print("%s [%s, %s], %s\n", op+1, REG(8, 6), REG(5, 3), REG(2, 0)); - else - print("%s %s, [%s, %s]\n", op+1, REG(2, 0), REG(8, 6), REG(5, 3)); - return; - case 3: - op = OP7(12, 11); - if(B(12, 11) == 0 || B(12,11) == 1) - l = 4; - else - l = 1; - if(*op == 'l') - print("%s %d(%s), %s\n", op+1, l*IMM(10, 6), REG(5, 3), REG(2, 0)); - else - print("%s %s, %d(%s)\n", op+1, REG(2, 0), l*IMM(10, 6), REG(5, 3)); - return; - case 4: - if(B(12, 12) == 0){ - op = OP8(11, 11); - if(*op == 'l') - print("%s %d(%s), %s\n", op+1, 2*IMM(10, 6), REG(5, 3), REG(2, 0)); - else - print("%s %s, %d(%s)\n", op+1, REG(2, 0), 2*IMM(10, 6), REG(5, 3)); - return; - } - op = OP9(11, 11); - if(*op == 'l') - print("%s %d(sp), %s\n", op+1, 4*IMM(7, 0), REG(10, 8)); - else - print("%s %s, %d(sp)\n", op+1, REG(10, 8), 4*IMM(7, 0)); - return; - case 5: - if(B(12, 12) == 0){ - if(B(11, 11) == 0) - print("add %d, pc, %s\n", 4*IMM(7, 0), REG(10, 8)); - else - print("add %d, sp, %s\n", 4*IMM(7, 0), REG(10, 8)); - return; - } - if(B(11, 8) == 0){ - print("%s %d, sp\n", OP2(7, 7), 4*IMM(6, 0)); - return; - } - SBO(10, 10); - SBZ(9, 9); - if(B(8, 8) == 0) - print("%s sp, %d\n", OP10(11, 11), IMM(7, 0)); - else - print("%s sp, %d|15\n", OP10(11, 11), IMM(7, 0)); - return; - case 6: - if(B(12, 12) == 0){ - print("%s %s, %d\n", OP11(11, 11), REG(10, 8), IMM(7, 0)); - return; - } - if(B(11, 8) == 0xf){ - print("swi %d\n", IMM(7, 0)); - return; - } - o = IMM(7, 0); - if(o&0x80) - o |= 0xffffff00; - o = pc+4+(o<<1); - print("b%s %x\n", COND(11, 8), o); - return; - case 7: - o = B(12, 11); - switch(o){ - case 0: - o = IMM(10, 0); - if(o&0x400) - o |= 0xfffff800; - o = pc+4+(o<<1); - print("b %x\n", o); - return; - case 1: - illegal(i, pc); - return; - case 2: - lasto = IMM(10, 0); - print("bl\n"); - return; - case 3: - if(lasto&0x400) - lasto |= 0xfffff800; - o = IMM(10, 0); - o = (pc-2)+4+(o<<1)+(lasto<<12); - print("bl %x\n", o); - return; - } - } -} diff --git a/src/cmd/6g/Makefile b/src/cmd/6g/Makefile index 023f5d111..64fa15399 100644 --- a/src/cmd/6g/Makefile +++ b/src/cmd/6g/Makefile @@ -14,16 +14,17 @@ HFILES=\ opt.h\ OFILES=\ - list.$O\ - gobj.$O\ - galign.$O\ - ggen.$O\ + ../6l/enam.$O\ cgen.$O\ cplx.$O\ + galign.$O\ + ggen.$O\ + gobj.$O\ gsubr.$O\ + list.$O\ peep.$O\ + pgen.$O\ reg.$O\ - ../6l/enam.$O\ LIB=\ ../gc/gc.a\ diff --git a/src/cmd/6g/cgen.c b/src/cmd/6g/cgen.c index 75dc4fe13..fca4b64dd 100644 --- a/src/cmd/6g/cgen.c +++ b/src/cmd/6g/cgen.c @@ -283,11 +283,9 @@ cgen(Node *n, Node *res) if(istype(nl->type, TSTRING) || isslice(nl->type)) { // both slice and string have len one pointer into the struct. // a zero pointer means zero length - regalloc(&n1, types[tptr], res); - agen(nl, &n1); - n1.op = OINDREG; + igen(nl, &n1, res); n1.type = types[TUINT32]; - n1.xoffset = Array_nel; + n1.xoffset += Array_nel; gmove(&n1, res); regfree(&n1); break; @@ -319,11 +317,9 @@ cgen(Node *n, Node *res) break; } if(isslice(nl->type)) { - regalloc(&n1, types[tptr], res); - agen(nl, &n1); - n1.op = OINDREG; + igen(nl, &n1, res); n1.type = types[TUINT32]; - n1.xoffset = Array_cap; + n1.xoffset += Array_cap; gmove(&n1, res); regfree(&n1); break; @@ -542,7 +538,8 @@ agen(Node *n, Node *res) gmove(&n1, &n3); } - ginscon(optoas(OADD, types[tptr]), v*w, &n3); + if (v*w != 0) + ginscon(optoas(OADD, types[tptr]), v*w, &n3); gmove(&n3, res); regfree(&n3); break; @@ -682,6 +679,28 @@ ret: void igen(Node *n, Node *a, Node *res) { + Type *fp; + Iter flist; + + switch(n->op) { + case ONAME: + if((n->class&PHEAP) || n->class == PPARAMREF) + break; + *a = *n; + return; + + case OCALLFUNC: + fp = structfirst(&flist, getoutarg(n->left->type)); + cgen_call(n, 0); + memset(a, 0, sizeof *a); + a->op = OINDREG; + a->val.u.reg = D_SP; + a->addable = 1; + a->xoffset = fp->width; + a->type = n->type; + return; + } + regalloc(a, types[tptr], res); agen(n, a); a->op = OINDREG; @@ -848,6 +867,7 @@ bgen(Node *n, int true, Prog *to) n2 = n1; n2.op = OINDREG; n2.xoffset = Array_array; + n2.type = types[tptr]; nodconst(&tmp, types[tptr], 0); gins(optoas(OCMP, types[tptr]), &n2, &tmp); patch(gbranch(a, types[tptr]), to); diff --git a/src/cmd/6g/gg.h b/src/cmd/6g/gg.h index 7efb2c252..2493771a0 100644 --- a/src/cmd/6g/gg.h +++ b/src/cmd/6g/gg.h @@ -23,6 +23,7 @@ struct Addr Sym* gotype; Sym* sym; + Node* node; int width; uchar type; uchar index; diff --git a/src/cmd/6g/ggen.c b/src/cmd/6g/ggen.c index 8d89fb164..5260335df 100644 --- a/src/cmd/6g/ggen.c +++ b/src/cmd/6g/ggen.c @@ -7,121 +7,18 @@ #include "gg.h" #include "opt.h" -static Prog *pret; - void -compile(Node *fn) +defframe(Prog *ptxt) { - Plist *pl; - Node nod1; - Prog *ptxt; - int32 lno; - Type *t; - Iter save; - - if(newproc == N) { - newproc = sysfunc("newproc"); - deferproc = sysfunc("deferproc"); - deferreturn = sysfunc("deferreturn"); - panicindex = sysfunc("panicindex"); - panicslice = sysfunc("panicslice"); - throwreturn = sysfunc("throwreturn"); - } - - if(fn->nbody == nil) - return; - - // set up domain for labels - clearlabels(); - - lno = setlineno(fn); - - curfn = fn; - dowidth(curfn->type); - - if(curfn->type->outnamed) { - // add clearing of the output parameters - t = structfirst(&save, getoutarg(curfn->type)); - while(t != T) { - if(t->nname != N) - curfn->nbody = concat(list1(nod(OAS, t->nname, N)), curfn->nbody); - t = structnext(&save); - } - } - - hasdefer = 0; - walk(curfn); - if(nerrors != 0 || isblank(curfn->nname)) - goto ret; - - allocparams(); - - continpc = P; - breakpc = P; - - pl = newplist(); - pl->name = curfn->nname; - - setlineno(curfn); - - nodconst(&nod1, types[TINT32], 0); - ptxt = gins(ATEXT, curfn->nname, &nod1); - afunclit(&ptxt->from); - - ginit(); - genlist(curfn->enter); - - pret = nil; - if(hasdefer || curfn->exit) { - Prog *p1; - - p1 = gjmp(nil); - pret = gjmp(nil); - patch(p1, pc); - } - - genlist(curfn->nbody); - gclean(); - checklabels(); - if(nerrors != 0) - goto ret; - if(curfn->endlineno) - lineno = curfn->endlineno; - - if(curfn->type->outtuple != 0) - ginscall(throwreturn, 0); - - if(pret) - patch(pret, pc); - ginit(); - if(hasdefer) - ginscall(deferreturn, 0); - if(curfn->exit) - genlist(curfn->exit); - gclean(); - if(nerrors != 0) - goto ret; - pc->as = ARET; // overwrite AEND - pc->lineno = lineno; - - if(!debug['N'] || debug['R'] || debug['P']) { - regopt(ptxt); - } - // fill in argument size ptxt->to.offset = rnd(curfn->type->argwid, widthptr); // fill in final stack size ptxt->to.offset <<= 32; ptxt->to.offset |= rnd(stksize+maxarg, widthptr); - - if(debug['f']) - frame(0); - -ret: - lineno = lno; } + /* * generate: * call f @@ -163,7 +60,7 @@ ginscall(Node *f, int proc) if(proc == 2) { nodreg(®, types[TINT64], D_AX); gins(ATESTQ, ®, ®); - patch(gbranch(AJNE, T), pret); + patch(gbranch(AJNE, T), retpc); } break; } @@ -201,7 +98,7 @@ cgen_callinter(Node *n, Node *res, int proc) regalloc(&nodo, types[tptr], &nodr); nodo.op = OINDREG; - agen(i, &nodr); // REG = &inter + agen(i, &nodr); // REG = &inter nodindreg(&nodsp, types[tptr], D_SP); nodo.xoffset += widthptr; @@ -355,7 +252,7 @@ cgen_ret(Node *n) { genlist(n->list); // copy out args if(hasdefer || curfn->exit) - gjmp(pret); + gjmp(retpc); else gins(ARET, N, N); } @@ -1206,7 +1103,7 @@ cgen_inline(Node *n, Node *res) Node nodes[5]; Node n1, n2, nres, ntemp; vlong v; - int i, narg; + int i, narg, nochk; if(n->op != OCALLFUNC) goto no; @@ -1242,6 +1139,7 @@ slicearray: // len = hb[3] - lb[2] (destroys hb) n2 = *res; n2.xoffset += Array_nel; + n2.type = types[TUINT32]; if(smallintconst(&nodes[3]) && smallintconst(&nodes[2])) { v = mpgetfix(nodes[3].val.u.xval) - @@ -1260,6 +1158,7 @@ slicearray: // cap = nel[1] - lb[2] (destroys nel) n2 = *res; n2.xoffset += Array_cap; + n2.type = types[TUINT32]; if(smallintconst(&nodes[1]) && smallintconst(&nodes[2])) { v = mpgetfix(nodes[1].val.u.xval) - @@ -1288,6 +1187,7 @@ slicearray: // ary = old[0] + (lb[2] * width[4]) (destroys old) n2 = *res; n2.xoffset += Array_array; + n2.type = types[tptr]; if(smallintconst(&nodes[2]) && smallintconst(&nodes[4])) { v = mpgetfix(nodes[2].val.u.xval) * @@ -1311,6 +1211,7 @@ slicearray: return 1; sliceslice: + nochk = n->etype; // skip bounds checking ntemp.op = OXXX; if(!sleasy(n->list->n->right)) { Node *n0; @@ -1340,11 +1241,13 @@ sliceslice: n2 = nodes[0]; n2.xoffset += Array_nel; n2.type = types[TUINT32]; - cmpandthrow(&nodes[1], &n2); + if(!nochk) + cmpandthrow(&nodes[1], &n2); // ret.nel = old.nel[0]-lb[1]; n2 = nodes[0]; n2.xoffset += Array_nel; + n2.type = types[TUINT32]; regalloc(&n1, types[TUINT32], N); gins(optoas(OAS, types[TUINT32]), &n2, &n1); @@ -1353,22 +1256,24 @@ sliceslice: n2 = nres; n2.xoffset += Array_nel; + n2.type = types[TUINT32]; gins(optoas(OAS, types[TUINT32]), &n1, &n2); regfree(&n1); } else { // old[lb:hb] - // if(hb[2] > old.cap[0]) goto throw; n2 = nodes[0]; n2.xoffset += Array_cap; n2.type = types[TUINT32]; - cmpandthrow(&nodes[2], &n2); - - // if(lb[1] > hb[2]) goto throw; - cmpandthrow(&nodes[1], &nodes[2]); - + if(!nochk) { + // if(hb[2] > old.cap[0]) goto throw; + cmpandthrow(&nodes[2], &n2); + // if(lb[1] > hb[2]) goto throw; + cmpandthrow(&nodes[1], &nodes[2]); + } // ret.len = hb[2]-lb[1]; (destroys hb[2]) n2 = nres; n2.xoffset += Array_nel; - + n2.type = types[TUINT32]; + if(smallintconst(&nodes[2]) && smallintconst(&nodes[1])) { v = mpgetfix(nodes[2].val.u.xval) - mpgetfix(nodes[1].val.u.xval); @@ -1387,6 +1292,7 @@ sliceslice: // ret.cap = old.cap[0]-lb[1]; (uses hb[2]) n2 = nodes[0]; n2.xoffset += Array_cap; + n2.type = types[TUINT32]; regalloc(&n1, types[TUINT32], &nodes[2]); gins(optoas(OAS, types[TUINT32]), &n2, &n1); @@ -1395,13 +1301,15 @@ sliceslice: n2 = nres; n2.xoffset += Array_cap; + n2.type = types[TUINT32]; + gins(optoas(OAS, types[TUINT32]), &n1, &n2); regfree(&n1); // ret.array = old.array[0]+lb[1]*width[3]; (uses lb[1]) n2 = nodes[0]; n2.xoffset += Array_array; - + n2.type = types[tptr]; regalloc(&n1, types[tptr], &nodes[1]); if(smallintconst(&nodes[1]) && smallintconst(&nodes[3])) { gins(optoas(OAS, types[tptr]), &n2, &n1); @@ -1418,6 +1326,7 @@ sliceslice: n2 = nres; n2.xoffset += Array_array; + n2.type = types[tptr]; gins(optoas(OAS, types[tptr]), &n1, &n2); regfree(&n1); diff --git a/src/cmd/6g/gsubr.c b/src/cmd/6g/gsubr.c index 78b8c4351..49d66e083 100644 --- a/src/cmd/6g/gsubr.c +++ b/src/cmd/6g/gsubr.c @@ -48,7 +48,7 @@ clearp(Prog *p) /* * generate and return proc with p->as = as, - * linked into program. pc is next instruction. + * linked into program. pc is next instruction. */ Prog* prog(int as) @@ -368,11 +368,13 @@ regfree(Node *n) { int i; - if(n->op == ONAME && iscomplex[n->type->etype]) + if(n->op == ONAME) return; if(n->op != OREGISTER && n->op != OINDREG) fatal("regfree: not a register"); i = n->val.u.reg; + if(i == D_SP) + return; if(i < 0 || i >= sizeof(reg)) fatal("regfree: reg out of range"); if(reg[i] <= 0) @@ -926,7 +928,7 @@ Prog* gins(int as, Node *f, Node *t) { // Node nod; -// int32 v; + int32 w; Prog *p; Addr af, at; @@ -971,6 +973,27 @@ gins(int as, Node *f, Node *t) p->to = at; if(debug['g']) print("%P\n", p); + + + w = 0; + switch(as) { + case AMOVB: + w = 1; + break; + case AMOVW: + w = 2; + break; + case AMOVL: + w = 4; + break; + case AMOVQ: + w = 8; + break; + } + if(w != 0 && f != N && (af.width > w || at.width > w)) { + fatal("bad width: %P (%d, %d)\n", p, af.width, at.width); + } + return p; } @@ -985,7 +1008,7 @@ checkoffset(Addr *a, int canemitcode) fatal("checkoffset %#llx, cannot emit code", a->offset); // cannot rely on unmapped nil page at 0 to catch - // reference with large offset. instead, emit explicit + // reference with large offset. instead, emit explicit // test of 0(reg). p = gins(ATESTB, nodintconst(0), N); p->to = *a; @@ -1003,7 +1026,7 @@ naddr(Node *n, Addr *a, int canemitcode) a->index = D_NONE; a->type = D_NONE; a->gotype = S; - + a->node = N; if(n == N) return; @@ -1082,6 +1105,8 @@ naddr(Node *n, Addr *a, int canemitcode) break; case PAUTO: a->type = D_AUTO; + if (n->sym) + a->node = n->orig; break; case PPARAM: case PPARAMOUT: @@ -1144,8 +1169,9 @@ naddr(Node *n, Addr *a, int canemitcode) naddr(n->left, a, canemitcode); if(a->type == D_CONST && a->offset == 0) break; // len(nil) - a->etype = TUINT; + a->etype = TUINT32; a->offset += Array_nel; + a->width = 4; if(a->offset >= unmappedzero && a->offset-Array_nel < unmappedzero) checkoffset(a, canemitcode); break; @@ -1155,8 +1181,9 @@ naddr(Node *n, Addr *a, int canemitcode) naddr(n->left, a, canemitcode); if(a->type == D_CONST && a->offset == 0) break; // cap(nil) - a->etype = TUINT; + a->etype = TUINT32; a->offset += Array_cap; + a->width = 4; if(a->offset >= unmappedzero && a->offset-Array_cap < unmappedzero) checkoffset(a, canemitcode); break; @@ -2000,12 +2027,12 @@ oindex: if(o & OAddable) { n2 = *l; n2.xoffset += Array_array; - n2.type = types[TUINT64]; + n2.type = types[tptr]; gmove(&n2, reg); } else { n2 = *reg; - n2.xoffset = Array_array; n2.op = OINDREG; + n2.xoffset = Array_array; n2.type = types[tptr]; gmove(&n2, reg); } diff --git a/src/cmd/6g/reg.c b/src/cmd/6g/reg.c index ed8bac3f0..af9b29cbc 100644 --- a/src/cmd/6g/reg.c +++ b/src/cmd/6g/reg.c @@ -33,6 +33,8 @@ #define EXTERN #include "opt.h" +#define NREGVAR 32 /* 16 general + 16 floating */ +#define REGBITS ((uint32)0xffffffff) #define P2R(p) (Reg*)(p->reg) static int first = 1; @@ -114,6 +116,41 @@ setaddrs(Bits bit) } } +static char* regname[] = { + ".AX", + ".CX", + ".DX", + ".BX", + ".SP", + ".BP", + ".SI", + ".DI", + ".R8", + ".R9", + ".R10", + ".R11", + ".R12", + ".R13", + ".R14", + ".R15", + ".X0", + ".X1", + ".X2", + ".X3", + ".X4", + ".X5", + ".X6", + ".X7", + ".X8", + ".X9", + ".X10", + ".X11", + ".X12", + ".X13", + ".X14", + ".X15", +}; + void regopt(Prog *firstp) { @@ -143,6 +180,17 @@ regopt(Prog *firstp) firstr = R; lastr = R; nvar = 0; + + /* + * control flow is more complicated in generated go code + * than in generated c code. define pseudo-variables for + * registers, so we have complete register usage information. + */ + nvar = NREGVAR; + memset(var, 0, NREGVAR*sizeof var[0]); + for(i=0; i<NREGVAR; i++) + var[i].sym = lookup(regname[i]); + regbits = RtoB(D_SP); for(z=0; z<BITS; z++) { externs.b[z] = 0; @@ -247,6 +295,9 @@ regopt(Prog *firstp) case ACOMISD: case AUCOMISS: case AUCOMISD: + case ATESTB: + case ATESTL: + case ATESTQ: for(z=0; z<BITS; z++) r->use2.b[z] |= bit.b[z]; break; @@ -254,6 +305,7 @@ regopt(Prog *firstp) /* * right side write */ + case ALEAQ: case ANOP: case AMOVL: case AMOVQ: @@ -261,6 +313,8 @@ regopt(Prog *firstp) case AMOVW: case AMOVBLSX: case AMOVBLZX: + case AMOVBWSX: + case AMOVBWZX: case AMOVBQSX: case AMOVBQZX: case AMOVLQSX: @@ -269,6 +323,7 @@ regopt(Prog *firstp) case AMOVWLZX: case AMOVWQSX: case AMOVWQZX: + case APOPQ: case AMOVSS: case AMOVSD: @@ -357,6 +412,8 @@ regopt(Prog *firstp) case AIMULL: case AIMULQ: case AIMULW: + case ANEGB: + case ANEGW: case ANEGL: case ANEGQ: case ANOTL: @@ -366,6 +423,23 @@ regopt(Prog *firstp) case ASBBL: case ASBBQ: + case ASETCC: + case ASETCS: + case ASETEQ: + case ASETGE: + case ASETGT: + case ASETHI: + case ASETLE: + case ASETLS: + case ASETLT: + case ASETMI: + case ASETNE: + case ASETOC: + case ASETOS: + case ASETPC: + case ASETPL: + case ASETPS: + case AXCHGB: case AXCHGW: case AXCHGL: @@ -411,32 +485,44 @@ regopt(Prog *firstp) if(p->to.type != D_NONE) break; - case AIDIVB: case AIDIVL: - case AIDIVQ: case AIDIVW: - case AIMULB: - case ADIVB: + case AIDIVQ: case ADIVL: - case ADIVQ: case ADIVW: - case AMULB: + case ADIVQ: case AMULL: - case AMULQ: case AMULW: + case AMULQ: + r->set.b[0] |= RtoB(D_AX) | RtoB(D_DX); + r->use1.b[0] |= RtoB(D_AX) | RtoB(D_DX); + break; + + case AIDIVB: + case AIMULB: + case ADIVB: + case AMULB: + r->set.b[0] |= RtoB(D_AX); + r->use1.b[0] |= RtoB(D_AX); + break; case ACWD: - case ACDQ: - case ACQO: - r->regu |= RtoB(D_AX) | RtoB(D_DX); + r->set.b[0] |= RtoB(D_AX) | RtoB(D_DX); + r->use1.b[0] |= RtoB(D_AX); break; + case ACDQ: + r->set.b[0] |= RtoB(D_DX); + r->use1.b[0] |= RtoB(D_AX); + break; + case AREP: case AREPN: case ALOOP: case ALOOPEQ: case ALOOPNE: - r->regu |= RtoB(D_CX); + r->set.b[0] |= RtoB(D_CX); + r->use1.b[0] |= RtoB(D_CX); break; case AMOVSB: @@ -447,7 +533,8 @@ regopt(Prog *firstp) case ACMPSL: case ACMPSQ: case ACMPSW: - r->regu |= RtoB(D_SI) | RtoB(D_DI); + r->set.b[0] |= RtoB(D_SI) | RtoB(D_DI); + r->use1.b[0] |= RtoB(D_SI) | RtoB(D_DI); break; case ASTOSB: @@ -458,16 +545,22 @@ regopt(Prog *firstp) case ASCASL: case ASCASQ: case ASCASW: - r->regu |= RtoB(D_AX) | RtoB(D_DI); + r->set.b[0] |= RtoB(D_DI); + r->use1.b[0] |= RtoB(D_AX) | RtoB(D_DI); break; case AINSB: case AINSL: case AINSW: + r->set.b[0] |= RtoB(D_DX) | RtoB(D_DI); + r->use1.b[0] |= RtoB(D_DI); + break; + case AOUTSB: case AOUTSL: case AOUTSW: - r->regu |= RtoB(D_DI) | RtoB(D_DX); + r->set.b[0] |= RtoB(D_DI); + r->use1.b[0] |= RtoB(D_DX) | RtoB(D_DI); break; } } @@ -574,6 +667,24 @@ loop2: dumpit("pass4", firstr); /* + * pass 4.5 + * move register pseudo-variables into regu. + */ + for(r = firstr; r != R; r = r->link) { + r->regu = (r->refbehind.b[0] | r->set.b[0]) & REGBITS; + + r->set.b[0] &= ~REGBITS; + r->use1.b[0] &= ~REGBITS; + r->use2.b[0] &= ~REGBITS; + r->refbehind.b[0] &= ~REGBITS; + r->refahead.b[0] &= ~REGBITS; + r->calbehind.b[0] &= ~REGBITS; + r->calahead.b[0] &= ~REGBITS; + r->regdiff.b[0] &= ~REGBITS; + r->act.b[0] &= ~REGBITS; + } + + /* * pass 5 * isolate regions * calculate costs (paint1) @@ -726,6 +837,7 @@ addmove(Reg *r, int bn, int rn, int f) a->etype = v->etype; a->type = v->name; a->gotype = v->gotype; + a->node = v->node; // need to clean this up with wptr and // some of the defaults @@ -818,6 +930,7 @@ mkvar(Reg *r, Adr *a) { Var *v; int i, t, n, et, z, w, flag; + uint32 regu; int32 o; Bits bit; Sym *s; @@ -829,14 +942,17 @@ mkvar(Reg *r, Adr *a) if(t == D_NONE) goto none; - if(r != R) { - r->regu |= doregbits(t); - r->regu |= doregbits(a->index); - } + if(r != R) + r->use1.b[0] |= doregbits(a->index); switch(t) { default: - goto none; + regu = doregbits(t); + if(regu == 0) + goto none; + bit = zbits; + bit.b[0] = regu; + return bit; case D_ADDR: a->type = a->index; @@ -873,14 +989,17 @@ mkvar(Reg *r, Adr *a) // if they overlaps, disable both if(overlap(v->offset, v->width, o, w)) { +// print("disable overlap %s %d %d %d %d, %E != %E\n", s->name, v->offset, v->width, o, w, v->etype, et); v->addr = 1; flag = 1; } } } - if(a->pun) + if(a->pun) { +// print("disable pun %s\n", s->name); flag = 1; + } switch(et) { case 0: case TFUNC: @@ -903,6 +1022,7 @@ mkvar(Reg *r, Adr *a) v->etype = et; v->width = w; v->addr = flag; // funny punning + v->node = a->node; if(debug['R']) print("bit=%2d et=%2d w=%d %S %D\n", i, et, w, s, a); diff --git a/src/cmd/6l/asm.c b/src/cmd/6l/asm.c index dda19e48d..4c04112b7 100644 --- a/src/cmd/6l/asm.c +++ b/src/cmd/6l/asm.c @@ -108,6 +108,9 @@ needlib(char *name) char *p; Sym *s; + if(*name == '\0') + return 0; + /* reuse hash code in symbol table */ p = smprint(".elfload.%s", name); s = lookup(p, 0); @@ -695,7 +698,7 @@ asmb(void) { int32 magic; int a, dynsym; - vlong vl, startva, symo, elfsymo, elfstro, elfsymsize, machlink; + vlong vl, startva, symo, machlink; ElfEhdr *eh; ElfPhdr *ph, *pph; ElfShdr *sh; @@ -706,9 +709,6 @@ asmb(void) Bflush(&bso); elftextsh = 0; - elfsymsize = 0; - elfstro = 0; - elfsymo = 0; if(debug['v']) Bprint(&bso, "%5.2f codeblk\n", cputime()); @@ -787,40 +787,28 @@ asmb(void) symo = rnd(symo, PEFILEALIGN); break; } - /* - * the symbol information is stored as - * 32-bit symbol table size - * 32-bit line number table size - * symbol table - * line number table - */ - seek(cout, symo+8, 0); - if(debug['v']) - Bprint(&bso, "%5.2f sp\n", cputime()); - Bflush(&bso); - if(debug['v']) - Bprint(&bso, "%5.2f pc\n", cputime()); - Bflush(&bso); - if(!debug['s']) - strnput("", INITRND-(8+symsize+lcsize)%INITRND); - cflush(); seek(cout, symo, 0); - lputl(symsize); - lputl(lcsize); - cflush(); - if(HEADTYPE != Hwindows && !debug['s']) { - elfsymo = symo+8+symsize+lcsize; - seek(cout, elfsymo, 0); - asmelfsym64(); - cflush(); - elfstro = seek(cout, 0, 1); - elfsymsize = elfstro - elfsymo; - ewrite(cout, elfstrdat, elfstrsize); + switch(HEADTYPE) { + default: + if(iself) { + seek(cout, symo, 0); + asmelfsym(); + cflush(); + ewrite(cout, elfstrdat, elfstrsize); + + if(debug['v']) + Bprint(&bso, "%5.2f dwarf\n", cputime()); + dwarfemitdebugsections(); + } + break; + case Hdarwin: + case Hwindows: if(debug['v']) Bprint(&bso, "%5.2f dwarf\n", cputime()); dwarfemitdebugsections(); + break; } } @@ -1039,15 +1027,15 @@ asmb(void) sh = newElfShdr(elfstr[ElfStrSymtab]); sh->type = SHT_SYMTAB; - sh->off = elfsymo; - sh->size = elfsymsize; + sh->off = symo; + sh->size = symsize; sh->addralign = 8; sh->entsize = 24; sh->link = eh->shnum; // link to strtab sh = newElfShdr(elfstr[ElfStrStrtab]); sh->type = SHT_STRTAB; - sh->off = elfstro; + sh->off = symo+symsize; sh->size = elfstrsize; sh->addralign = 1; @@ -1134,6 +1122,10 @@ genasmsym(void (*put)(Sym*, char*, int, vlong, vlong, int, Sym*)) Auto *a; Sym *s; + s = lookup("etext", 0); + if(s->type == STEXT) + put(s, s->name, 'T', s->value, s->size, s->version, 0); + for(s=allsym; s!=S; s=s->allsym) { if(s->hide) continue; diff --git a/src/cmd/6l/l.h b/src/cmd/6l/l.h index 33ca51b2c..f4ee6aa92 100644 --- a/src/cmd/6l/l.h +++ b/src/cmd/6l/l.h @@ -350,9 +350,6 @@ EXTERN Sym* fromgotype; // type symbol on last p->from read EXTERN vlong textstksiz; EXTERN vlong textarg; -EXTERN int elfstrsize; -EXTERN char* elfstrdat; -EXTERN int elftextsh; extern Optab optab[]; extern Optab* opindex[]; diff --git a/src/cmd/6l/obj.c b/src/cmd/6l/obj.c index d53814a74..e3191bb4d 100644 --- a/src/cmd/6l/obj.c +++ b/src/cmd/6l/obj.c @@ -177,7 +177,7 @@ main(int argc, char *argv[]) */ tlsoffset = 0x8a0; machoinit(); - HEADR = MACHORESERVE; + HEADR = INITIAL_MACHO_HEADR; if(INITRND == -1) INITRND = 4096; if(INITTEXT == -1) diff --git a/src/cmd/8g/Makefile b/src/cmd/8g/Makefile index 09cf8d4e3..b459782a3 100644 --- a/src/cmd/8g/Makefile +++ b/src/cmd/8g/Makefile @@ -15,15 +15,16 @@ HFILES=\ OFILES=\ ../8l/enam.$O\ - list.$O\ - galign.$O\ - gobj.$O\ - ggen.$O\ - gsubr.$O\ cgen.$O\ cgen64.$O\ cplx.$O\ + galign.$O\ + ggen.$O\ + gobj.$O\ + gsubr.$O\ + list.$O\ peep.$O\ + pgen.$O\ reg.$O\ LIB=\ diff --git a/src/cmd/8g/cgen.c b/src/cmd/8g/cgen.c index 596824a6c..1614a2d77 100644 --- a/src/cmd/8g/cgen.c +++ b/src/cmd/8g/cgen.c @@ -232,6 +232,7 @@ cgen(Node *n, Node *res) cgen(nl, res); break; } + tempname(&n2, n->type); mgen(nl, &n1, res); gmove(&n1, &n2); @@ -277,15 +278,10 @@ cgen(Node *n, Node *res) if(istype(nl->type, TSTRING) || isslice(nl->type)) { // both slice and string have len one pointer into the struct. igen(nl, &n1, res); - n1.op = OREGISTER; // was OINDREG - regalloc(&n2, types[TUINT32], &n1); - n1.op = OINDREG; n1.type = types[TUINT32]; - n1.xoffset = Array_nel; - gmove(&n1, &n2); - gmove(&n2, res); + n1.xoffset += Array_nel; + gmove(&n1, res); regfree(&n1); - regfree(&n2); break; } fatal("cgen: OLEN: unknown type %lT", nl->type); @@ -594,9 +590,10 @@ agen(Node *n, Node *res) gmove(&n1, &n3); } - nodconst(&n2, types[tptr], v*w); - gins(optoas(OADD, types[tptr]), &n2, &n3); - + if (v*w != 0) { + nodconst(&n2, types[tptr], v*w); + gins(optoas(OADD, types[tptr]), &n2, &n3); + } gmove(&n3, res); regfree(&n3); break; @@ -729,7 +726,27 @@ void igen(Node *n, Node *a, Node *res) { Node n1; - + Type *fp; + Iter flist; + + switch(n->op) { + case ONAME: + if((n->class&PHEAP) || n->class == PPARAMREF) + break; + *a = *n; + return; + + case OCALLFUNC: + fp = structfirst(&flist, getoutarg(n->left->type)); + cgen_call(n, 0); + memset(a, 0, sizeof *a); + a->op = OINDREG; + a->val.u.reg = D_SP; + a->addable = 1; + a->xoffset = fp->width; + a->type = n->type; + return; + } // release register for now, to avoid // confusing tempname. if(res != N && res->op == OREGISTER) @@ -919,6 +936,7 @@ bgen(Node *n, int true, Prog *to) n2 = n1; n2.op = OINDREG; n2.xoffset = Array_array; + n2.type = types[tptr]; nodconst(&tmp, types[tptr], 0); gins(optoas(OCMP, types[tptr]), &n2, &tmp); patch(gbranch(a, types[tptr]), to); diff --git a/src/cmd/8g/gg.h b/src/cmd/8g/gg.h index 57cd1b56b..7da60d767 100644 --- a/src/cmd/8g/gg.h +++ b/src/cmd/8g/gg.h @@ -25,6 +25,7 @@ struct Addr Sym* gotype; Sym* sym; + Node* node; int width; uchar type; uchar index; diff --git a/src/cmd/8g/ggen.c b/src/cmd/8g/ggen.c index 920725c3e..25adb38c0 100644 --- a/src/cmd/8g/ggen.c +++ b/src/cmd/8g/ggen.c @@ -7,106 +7,9 @@ #include "gg.h" #include "opt.h" -static Prog *pret; - void -compile(Node *fn) +defframe(Prog *ptxt) { - Plist *pl; - Node nod1; - Prog *ptxt; - int32 lno; - Type *t; - Iter save; - - if(newproc == N) { - newproc = sysfunc("newproc"); - deferproc = sysfunc("deferproc"); - deferreturn = sysfunc("deferreturn"); - panicindex = sysfunc("panicindex"); - panicslice = sysfunc("panicslice"); - throwreturn = sysfunc("throwreturn"); - } - - if(fn->nbody == nil) - return; - - // set up domain for labels - clearlabels(); - - lno = setlineno(fn); - - curfn = fn; - dowidth(curfn->type); - - if(curfn->type->outnamed) { - // add clearing of the output parameters - t = structfirst(&save, getoutarg(curfn->type)); - while(t != T) { - if(t->nname != N) - curfn->nbody = concat(list1(nod(OAS, t->nname, N)), curfn->nbody); - t = structnext(&save); - } - } - - hasdefer = 0; - walk(curfn); - if(nerrors != 0 || isblank(curfn->nname)) - goto ret; - - allocparams(); - - continpc = P; - breakpc = P; - - pl = newplist(); - pl->name = curfn->nname; - - setlineno(curfn); - - nodconst(&nod1, types[TINT32], 0); - ptxt = gins(ATEXT, curfn->nname, &nod1); - afunclit(&ptxt->from); - - ginit(); - genlist(curfn->enter); - - pret = nil; - if(hasdefer || curfn->exit) { - Prog *p1; - - p1 = gjmp(nil); - pret = gjmp(nil); - patch(p1, pc); - } - - genlist(curfn->nbody); - gclean(); - checklabels(); - if(nerrors != 0) - goto ret; - if(curfn->endlineno) - lineno = curfn->endlineno; - - if(curfn->type->outtuple != 0) - ginscall(throwreturn, 0); - - if(pret) - patch(pret, pc); - ginit(); - if(hasdefer) - ginscall(deferreturn, 0); - if(curfn->exit) - genlist(curfn->exit); - gclean(); - if(nerrors != 0) - goto ret; - pc->as = ARET; // overwrite AEND - pc->lineno = lineno; - - if(!debug['N'] || debug['R'] || debug['P']) { - regopt(ptxt); - } // fill in argument size ptxt->to.offset2 = rnd(curfn->type->argwid, widthptr); @@ -115,12 +18,6 @@ compile(Node *fn) maxstksize = stksize; ptxt->to.offset = rnd(maxstksize+maxarg, widthptr); maxstksize = 0; - - if(debug['f']) - frame(0); - -ret: - lineno = lno; } void @@ -200,7 +97,7 @@ ginscall(Node *f, int proc) if(proc == 2) { nodreg(®, types[TINT64], D_AX); gins(ATESTL, ®, ®); - patch(gbranch(AJNE, T), pret); + patch(gbranch(AJNE, T), retpc); } break; } @@ -391,8 +288,8 @@ void cgen_ret(Node *n) { genlist(n->list); // copy out args - if(pret) - gjmp(pret); + if(retpc) + gjmp(retpc); else gins(ARET, N, N); } @@ -915,7 +812,7 @@ cgen_inline(Node *n, Node *res) Node nodes[5]; Node n1, n2, nres, ntemp; vlong v; - int i, narg; + int i, narg, nochk; if(n->op != OCALLFUNC) goto no; @@ -953,6 +850,7 @@ slicearray: // len = hb[3] - lb[2] (destroys hb) n2 = *res; n2.xoffset += Array_nel; + n2.type = types[TUINT32]; if(smallintconst(&nodes[3]) && smallintconst(&nodes[2])) { v = mpgetfix(nodes[3].val.u.xval) - @@ -971,6 +869,7 @@ slicearray: // cap = nel[1] - lb[2] (destroys nel) n2 = *res; n2.xoffset += Array_cap; + n2.type = types[TUINT32]; if(smallintconst(&nodes[1]) && smallintconst(&nodes[2])) { v = mpgetfix(nodes[1].val.u.xval) - @@ -999,6 +898,7 @@ slicearray: // ary = old[0] + (lb[2] * width[4]) (destroys old) n2 = *res; n2.xoffset += Array_array; + n2.type = types[tptr]; if(smallintconst(&nodes[2]) && smallintconst(&nodes[4])) { v = mpgetfix(nodes[2].val.u.xval) * @@ -1026,6 +926,7 @@ slicearray: sliceslice: if(!fix64(n->list, narg)) goto no; + nochk = n->etype; // skip bounds checking ntemp.op = OXXX; if(!sleasy(n->list->n->right)) { Node *n0; @@ -1055,11 +956,13 @@ sliceslice: n2 = nodes[0]; n2.xoffset += Array_nel; n2.type = types[TUINT32]; - cmpandthrow(&nodes[1], &n2); + if(!nochk) + cmpandthrow(&nodes[1], &n2); // ret.nel = old.nel[0]-lb[1]; n2 = nodes[0]; n2.xoffset += Array_nel; + n2.type = types[TUINT32]; regalloc(&n1, types[TUINT32], N); gins(optoas(OAS, types[TUINT32]), &n2, &n1); @@ -1068,22 +971,25 @@ sliceslice: n2 = nres; n2.xoffset += Array_nel; + n2.type = types[TUINT32]; gins(optoas(OAS, types[TUINT32]), &n1, &n2); regfree(&n1); } else { // old[lb:hb] - // if(hb[2] > old.cap[0]) goto throw; n2 = nodes[0]; n2.xoffset += Array_cap; n2.type = types[TUINT32]; - cmpandthrow(&nodes[2], &n2); - - // if(lb[1] > hb[2]) goto throw; - cmpandthrow(&nodes[1], &nodes[2]); + if (!nochk) { + // if(hb[2] > old.cap[0]) goto throw; + cmpandthrow(&nodes[2], &n2); + // if(lb[1] > hb[2]) goto throw; + cmpandthrow(&nodes[1], &nodes[2]); + } // ret.len = hb[2]-lb[1]; (destroys hb[2]) n2 = nres; n2.xoffset += Array_nel; - + n2.type = types[TUINT32]; + if(smallintconst(&nodes[2]) && smallintconst(&nodes[1])) { v = mpgetfix(nodes[2].val.u.xval) - mpgetfix(nodes[1].val.u.xval); @@ -1102,6 +1008,7 @@ sliceslice: // ret.cap = old.cap[0]-lb[1]; (uses hb[2]) n2 = nodes[0]; n2.xoffset += Array_cap; + n2.type = types[TUINT32]; regalloc(&n1, types[TUINT32], &nodes[2]); gins(optoas(OAS, types[TUINT32]), &n2, &n1); @@ -1110,12 +1017,14 @@ sliceslice: n2 = nres; n2.xoffset += Array_cap; + n2.type = types[TUINT32]; gins(optoas(OAS, types[TUINT32]), &n1, &n2); regfree(&n1); // ret.array = old.array[0]+lb[1]*width[3]; (uses lb[1]) n2 = nodes[0]; n2.xoffset += Array_array; + n2.type = types[tptr]; regalloc(&n1, types[tptr], &nodes[1]); if(smallintconst(&nodes[1]) && smallintconst(&nodes[3])) { @@ -1135,6 +1044,7 @@ sliceslice: n2 = nres; n2.xoffset += Array_array; + n2.type = types[tptr]; gins(optoas(OAS, types[tptr]), &n1, &n2); regfree(&n1); diff --git a/src/cmd/8g/gsubr.c b/src/cmd/8g/gsubr.c index ae9901eeb..a8d65cf22 100644 --- a/src/cmd/8g/gsubr.c +++ b/src/cmd/8g/gsubr.c @@ -736,7 +736,6 @@ ginit(void) reg[i] = 1; for(i=D_AL; i<=D_DI; i++) reg[i] = 0; - for(i=0; i<nelem(resvd); i++) reg[resvd[i]]++; } @@ -827,6 +826,8 @@ err: return; out: + if (i == D_SP) + print("alloc SP\n"); if(reg[i] == 0) { regpc[i] = (ulong)__builtin_return_address(0); if(i == D_AX || i == D_CX || i == D_DX || i == D_SP) { @@ -842,10 +843,14 @@ void regfree(Node *n) { int i; - + + if(n->op == ONAME) + return; if(n->op != OREGISTER && n->op != OINDREG) fatal("regfree: not a register"); i = n->val.u.reg; + if(i == D_SP) + return; if(i < 0 || i >= sizeof(reg)) fatal("regfree: reg out of range"); if(reg[i] <= 0) @@ -1167,6 +1172,9 @@ gmove(Node *f, Node *t) case CASE(TINT8, TUINT8): case CASE(TUINT8, TINT8): case CASE(TUINT8, TUINT8): + a = AMOVB; + break; + case CASE(TINT16, TINT8): // truncate case CASE(TUINT16, TINT8): case CASE(TINT32, TINT8): @@ -1176,7 +1184,7 @@ gmove(Node *f, Node *t) case CASE(TINT32, TUINT8): case CASE(TUINT32, TUINT8): a = AMOVB; - break; + goto rsrc; case CASE(TINT64, TINT8): // truncate low word case CASE(TUINT64, TINT8): @@ -1184,7 +1192,7 @@ gmove(Node *f, Node *t) case CASE(TUINT64, TUINT8): split64(f, &flo, &fhi); nodreg(&r1, t->type, D_AX); - gins(AMOVB, &flo, &r1); + gmove(&flo, &r1); gins(AMOVB, &r1, t); splitclean(); return; @@ -1193,12 +1201,15 @@ gmove(Node *f, Node *t) case CASE(TINT16, TUINT16): case CASE(TUINT16, TINT16): case CASE(TUINT16, TUINT16): + a = AMOVW; + break; + case CASE(TINT32, TINT16): // truncate case CASE(TUINT32, TINT16): case CASE(TINT32, TUINT16): case CASE(TUINT32, TUINT16): a = AMOVW; - break; + goto rsrc; case CASE(TINT64, TINT16): // truncate low word case CASE(TUINT64, TINT16): @@ -1206,7 +1217,7 @@ gmove(Node *f, Node *t) case CASE(TUINT64, TUINT16): split64(f, &flo, &fhi); nodreg(&r1, t->type, D_AX); - gins(AMOVW, &flo, &r1); + gmove(&flo, &r1); gins(AMOVW, &r1, t); splitclean(); return; @@ -1224,7 +1235,7 @@ gmove(Node *f, Node *t) case CASE(TUINT64, TUINT32): split64(f, &flo, &fhi); nodreg(&r1, t->type, D_AX); - gins(AMOVL, &flo, &r1); + gmove(&flo, &r1); gins(AMOVL, &r1, t); splitclean(); return; @@ -1378,14 +1389,14 @@ gmove(Node *f, Node *t) case TUINT8: gins(ATESTL, ncon(0xffffff00), &t1); p1 = gbranch(AJEQ, T); - gins(AMOVB, ncon(0), &t1); + gins(AMOVL, ncon(0), &t1); patch(p1, pc); gmove(&t1, t); break; case TUINT16: gins(ATESTL, ncon(0xffff0000), &t1); p1 = gbranch(AJEQ, T); - gins(AMOVW, ncon(0), &t1); + gins(AMOVL, ncon(0), &t1); patch(p1, pc); gmove(&t1, t); break; @@ -1456,11 +1467,11 @@ gmove(Node *f, Node *t) split64(t, &tlo, &thi); gins(AXORL, ncon(0x80000000), &thi); // + 2^63 patch(p3, pc); - patch(p1, pc); splitclean(); - // restore rounding mode gins(AFLDCW, &t1, N); + + patch(p1, pc); return; /* @@ -1609,6 +1620,14 @@ gmove(Node *f, Node *t) gins(a, f, t); return; +rsrc: + // requires register source + regalloc(&r1, f->type, t); + gmove(f, &r1); + gins(a, &r1, t); + regfree(&r1); + return; + rdst: // requires register destination regalloc(&r1, t->type, t); @@ -1661,6 +1680,7 @@ gins(int as, Node *f, Node *t) { Prog *p; Addr af, at; + int w; if(as == AFMOVF && f && f->op == OREGISTER && t && t->op == OREGISTER) fatal("gins MOVF reg, reg"); @@ -1686,6 +1706,26 @@ gins(int as, Node *f, Node *t) p->to = at; if(debug['g']) print("%P\n", p); + + w = 0; + switch(as) { + case AMOVB: + w = 1; + break; + case AMOVW: + w = 2; + break; + case AMOVL: + w = 4; + break; + } + + if(1 && w != 0 && f != N && (af.width > w || at.width > w)) { + dump("bad width from:", f); + dump("bad width to:", t); + fatal("bad width: %P (%d, %d)\n", p, af.width, at.width); + } + return p; } @@ -1718,6 +1758,7 @@ naddr(Node *n, Addr *a, int canemitcode) a->index = D_NONE; a->type = D_NONE; a->gotype = S; + a->node = N; if(n == N) return; @@ -1775,6 +1816,8 @@ naddr(Node *n, Addr *a, int canemitcode) break; case PAUTO: a->type = D_AUTO; + if (n->sym) + a->node = n->orig; break; case PPARAM: case PPARAMOUT: @@ -1837,8 +1880,9 @@ naddr(Node *n, Addr *a, int canemitcode) naddr(n->left, a, canemitcode); if(a->type == D_CONST && a->offset == 0) break; // len(nil) - a->etype = TUINT; + a->etype = TUINT32; a->offset += Array_nel; + a->width = 4; if(a->offset >= unmappedzero && a->offset-Array_nel < unmappedzero) checkoffset(a, canemitcode); break; @@ -1848,8 +1892,9 @@ naddr(Node *n, Addr *a, int canemitcode) naddr(n->left, a, canemitcode); if(a->type == D_CONST && a->offset == 0) break; // cap(nil) - a->etype = TUINT; + a->etype = TUINT32; a->offset += Array_cap; + a->width = 4; if(a->offset >= unmappedzero && a->offset-Array_nel < unmappedzero) checkoffset(a, canemitcode); break; diff --git a/src/cmd/8g/reg.c b/src/cmd/8g/reg.c index 1465d372c..a2f3def37 100644 --- a/src/cmd/8g/reg.c +++ b/src/cmd/8g/reg.c @@ -33,6 +33,8 @@ #define EXTERN #include "opt.h" +#define NREGVAR 8 +#define REGBITS ((uint32)0xff) #define P2R(p) (Reg*)(p->reg) static int first = 1; @@ -114,6 +116,8 @@ setaddrs(Bits bit) } } +static char* regname[] = { ".ax", ".cx", ".dx", ".bx", ".sp", ".bp", ".si", ".di" }; + void regopt(Prog *firstp) { @@ -142,7 +146,17 @@ regopt(Prog *firstp) r1 = R; firstr = R; lastr = R; - nvar = 0; + + /* + * control flow is more complicated in generated go code + * than in generated c code. define pseudo-variables for + * registers, so we have complete register usage information. + */ + nvar = NREGVAR; + memset(var, 0, NREGVAR*sizeof var[0]); + for(i=0; i<NREGVAR; i++) + var[i].sym = lookup(regname[i]); + regbits = RtoB(D_SP); for(z=0; z<BITS; z++) { externs.b[z] = 0; @@ -249,14 +263,19 @@ regopt(Prog *firstp) /* * right side write */ + case AFSTSW: + case ALEAL: case ANOP: case AMOVL: case AMOVB: case AMOVW: case AMOVBLSX: case AMOVBLZX: + case AMOVBWSX: + case AMOVBWZX: case AMOVWLSX: case AMOVWLZX: + case APOPL: for(z=0; z<BITS; z++) r->set.b[z] |= bit.b[z]; break; @@ -321,6 +340,23 @@ regopt(Prog *firstp) case AADCL: case ASBBL: + case ASETCC: + case ASETCS: + case ASETEQ: + case ASETGE: + case ASETGT: + case ASETHI: + case ASETLE: + case ASETLS: + case ASETLT: + case ASETMI: + case ASETNE: + case ASETOC: + case ASETOS: + case ASETPC: + case ASETPL: + case ASETPS: + case AXCHGB: case AXCHGW: case AXCHGL: @@ -349,20 +385,32 @@ regopt(Prog *firstp) if(p->to.type != D_NONE) break; - case AIDIVB: case AIDIVL: case AIDIVW: - case AIMULB: - case ADIVB: case ADIVL: case ADIVW: - case AMULB: case AMULL: case AMULW: + r->set.b[0] |= RtoB(D_AX) | RtoB(D_DX); + r->use1.b[0] |= RtoB(D_AX) | RtoB(D_DX); + break; + + case AIDIVB: + case AIMULB: + case ADIVB: + case AMULB: + r->set.b[0] |= RtoB(D_AX); + r->use1.b[0] |= RtoB(D_AX); + break; case ACWD: + r->set.b[0] |= RtoB(D_AX) | RtoB(D_DX); + r->use1.b[0] |= RtoB(D_AX); + break; + case ACDQ: - r->regu |= RtoB(D_AX) | RtoB(D_DX); + r->set.b[0] |= RtoB(D_DX); + r->use1.b[0] |= RtoB(D_AX); break; case AREP: @@ -370,7 +418,8 @@ regopt(Prog *firstp) case ALOOP: case ALOOPEQ: case ALOOPNE: - r->regu |= RtoB(D_CX); + r->set.b[0] |= RtoB(D_CX); + r->use1.b[0] |= RtoB(D_CX); break; case AMOVSB: @@ -379,7 +428,8 @@ regopt(Prog *firstp) case ACMPSB: case ACMPSL: case ACMPSW: - r->regu |= RtoB(D_SI) | RtoB(D_DI); + r->set.b[0] |= RtoB(D_SI) | RtoB(D_DI); + r->use1.b[0] |= RtoB(D_SI) | RtoB(D_DI); break; case ASTOSB: @@ -388,16 +438,22 @@ regopt(Prog *firstp) case ASCASB: case ASCASL: case ASCASW: - r->regu |= RtoB(D_AX) | RtoB(D_DI); + r->set.b[0] |= RtoB(D_DI); + r->use1.b[0] |= RtoB(D_AX) | RtoB(D_DI); break; case AINSB: case AINSL: case AINSW: + r->set.b[0] |= RtoB(D_DX) | RtoB(D_DI); + r->use1.b[0] |= RtoB(D_DI); + break; + case AOUTSB: case AOUTSL: case AOUTSW: - r->regu |= RtoB(D_DI) | RtoB(D_DX); + r->set.b[0] |= RtoB(D_DI); + r->use1.b[0] |= RtoB(D_DX) | RtoB(D_DI); break; } } @@ -504,6 +560,24 @@ loop2: dumpit("pass4", firstr); /* + * pass 4.5 + * move register pseudo-variables into regu. + */ + for(r = firstr; r != R; r = r->link) { + r->regu = (r->refbehind.b[0] | r->set.b[0]) & REGBITS; + + r->set.b[0] &= ~REGBITS; + r->use1.b[0] &= ~REGBITS; + r->use2.b[0] &= ~REGBITS; + r->refbehind.b[0] &= ~REGBITS; + r->refahead.b[0] &= ~REGBITS; + r->calbehind.b[0] &= ~REGBITS; + r->calahead.b[0] &= ~REGBITS; + r->regdiff.b[0] &= ~REGBITS; + r->act.b[0] &= ~REGBITS; + } + + /* * pass 5 * isolate regions * calculate costs (paint1) @@ -656,6 +730,7 @@ addmove(Reg *r, int bn, int rn, int f) a->etype = v->etype; a->type = v->name; a->gotype = v->gotype; + a->node = v->node; // need to clean this up with wptr and // some of the defaults @@ -732,7 +807,7 @@ Bits mkvar(Reg *r, Adr *a) { Var *v; - int i, t, n, et, z, w, flag; + int i, t, n, et, z, w, flag, regu; int32 o; Bits bit; Sym *s; @@ -744,14 +819,17 @@ mkvar(Reg *r, Adr *a) if(t == D_NONE) goto none; - if(r != R) { - r->regu |= doregbits(t); - r->regu |= doregbits(a->index); - } + if(r != R) + r->use1.b[0] |= doregbits(a->index); switch(t) { default: - goto none; + regu = doregbits(t); + if(regu == 0) + goto none; + bit = zbits; + bit.b[0] = regu; + return bit; case D_ADDR: a->type = a->index; @@ -821,6 +899,7 @@ mkvar(Reg *r, Adr *a) v->etype = et; v->width = w; v->addr = flag; // funny punning + v->node = a->node; if(debug['R']) print("bit=%2d et=%2d w=%d %S %D flag=%d\n", i, et, w, s, a, v->addr); diff --git a/src/cmd/8l/asm.c b/src/cmd/8l/asm.c index f28b8d904..cb900d28d 100644 --- a/src/cmd/8l/asm.c +++ b/src/cmd/8l/asm.c @@ -104,6 +104,9 @@ needlib(char *name) char *p; Sym *s; + if(*name == '\0') + return 0; + /* reuse hash code in symbol table */ p = smprint(".dynlib.%s", name); s = lookup(p, 0); @@ -532,6 +535,8 @@ doelf(void) elfstr[ElfStrGosymcounts] = addstring(shstrtab, ".gosymcounts"); elfstr[ElfStrGosymtab] = addstring(shstrtab, ".gosymtab"); elfstr[ElfStrGopclntab] = addstring(shstrtab, ".gopclntab"); + elfstr[ElfStrSymtab] = addstring(shstrtab, ".symtab"); + elfstr[ElfStrStrtab] = addstring(shstrtab, ".strtab"); dwarfaddshstrings(shstrtab); } elfstr[ElfStrShstrtab] = addstring(shstrtab, ".shstrtab"); @@ -715,10 +720,10 @@ asmb(void) if(iself) goto Elfsym; case Hgarbunix: - seek(cout, rnd(HEADR+segtext.filelen, 8192)+segdata.filelen, 0); + symo = rnd(HEADR+segtext.filelen, 8192)+segdata.filelen; break; case Hunixcoff: - seek(cout, rnd(HEADR+segtext.filelen, INITRND)+segdata.filelen, 0); + symo = rnd(HEADR+segtext.filelen, INITRND)+segdata.filelen; break; case Hplan9x32: symo = HEADR+segtext.filelen+segdata.filelen; @@ -736,32 +741,44 @@ asmb(void) symo = rnd(symo, INITRND); break; case Hwindows: - // TODO(brainman): not sure what symo meant to be, but it is not used for Windows PE for now anyway symo = rnd(HEADR+segtext.filelen, PEFILEALIGN)+segdata.filelen; symo = rnd(symo, PEFILEALIGN); break; } - if(!debug['s']) { - seek(cout, symo, 0); - - if(HEADTYPE == Hplan9x32) { - asmplan9sym(); + seek(cout, symo, 0); + switch(HEADTYPE) { + default: + if(iself) { + if(debug['v']) + Bprint(&bso, "%5.2f elfsym\n", cputime()); + asmelfsym(); cflush(); - - sym = lookup("pclntab", 0); - if(sym != nil) { - lcsize = sym->np; - for(i=0; i < lcsize; i++) - cput(sym->p[i]); - - cflush(); - } - - } else if(HEADTYPE != Hwindows) { + ewrite(cout, elfstrdat, elfstrsize); + if(debug['v']) Bprint(&bso, "%5.2f dwarf\n", cputime()); dwarfemitdebugsections(); } + break; + case Hplan9x32: + asmplan9sym(); + cflush(); + + sym = lookup("pclntab", 0); + if(sym != nil) { + lcsize = sym->np; + for(i=0; i < lcsize; i++) + cput(sym->p[i]); + + cflush(); + } + break; + case Hdarwin: + case Hwindows: + if(debug['v']) + Bprint(&bso, "%5.2f dwarf\n", cputime()); + dwarfemitdebugsections(); + break; } } if(debug['v']) @@ -1082,6 +1099,20 @@ asmb(void) sh->addralign = 1; shsym(sh, lookup("pclntab", 0)); + sh = newElfShdr(elfstr[ElfStrSymtab]); + sh->type = SHT_SYMTAB; + sh->off = symo; + sh->size = symsize; + sh->addralign = 4; + sh->entsize = 16; + sh->link = eh->shnum; // link to strtab + + sh = newElfShdr(elfstr[ElfStrStrtab]); + sh->type = SHT_STRTAB; + sh->off = symo+symsize; + sh->size = elfstrsize; + sh->addralign = 1; + dwarfaddelfheaders(); } diff --git a/src/cmd/8l/l.h b/src/cmd/8l/l.h index 8f39ef519..7e7cd5d63 100644 --- a/src/cmd/8l/l.h +++ b/src/cmd/8l/l.h @@ -260,11 +260,15 @@ EXTERN union #define cbuf u.obuf #define xbuf u.ibuf -#pragma varargck type "A" uint +#pragma varargck type "A" int #pragma varargck type "D" Adr* +#pragma varargck type "I" int +#pragma varargck type "I" uchar* #pragma varargck type "P" Prog* #pragma varargck type "R" int #pragma varargck type "S" char* +#pragma varargck type "Y" Sym* +#pragma varargck type "i" char* EXTERN int32 HEADR; EXTERN int32 HEADTYPE; @@ -383,11 +387,6 @@ void deadcode(void); #define WPUT(a) wputl(a) #define VPUT(a) vputl(a) -#pragma varargck type "D" Adr* -#pragma varargck type "P" Prog* -#pragma varargck type "R" int -#pragma varargck type "A" int - /* Used by ../ld/dwarf.c */ enum { diff --git a/src/cmd/8l/obj.c b/src/cmd/8l/obj.c index 2a38f7ef0..440dcb77f 100644 --- a/src/cmd/8l/obj.c +++ b/src/cmd/8l/obj.c @@ -188,7 +188,7 @@ main(int argc, char *argv[]) if(INITDAT == -1) INITDAT = 0; if(INITRND == -1) - INITRND = 1; + INITRND = 4096; break; case Hmsdoscom: /* MS-DOS .COM */ HEADR = 0; @@ -218,7 +218,7 @@ main(int argc, char *argv[]) */ tlsoffset = 0x468; machoinit(); - HEADR = MACHORESERVE; + HEADR = INITIAL_MACHO_HEADR; if(INITTEXT == -1) INITTEXT = 4096+HEADR; if(INITDAT == -1) diff --git a/src/cmd/8l/pass.c b/src/cmd/8l/pass.c index 28589b66a..72ae043d6 100644 --- a/src/cmd/8l/pass.c +++ b/src/cmd/8l/pass.c @@ -32,23 +32,10 @@ #include "l.h" #include "../ld/lib.h" +#include "../../pkg/runtime/stack.h" static void xfol(Prog*, Prog**); -// see ../../pkg/runtime/proc.c:/StackGuard -enum -{ -#ifdef __WINDOWS__ - // use larger stacks to compensate for larger stack guard, - // needed for exception handling. - StackSmall = 256, - StackBig = 8192, -#else - StackSmall = 128, - StackBig = 4096, -#endif -}; - Prog* brchain(Prog *p) { diff --git a/src/cmd/cgo/doc.go b/src/cmd/cgo/doc.go index b3aa9aded..064725c1d 100644 --- a/src/cmd/cgo/doc.go +++ b/src/cmd/cgo/doc.go @@ -35,9 +35,17 @@ systems. For example: // #include <png.h> import "C" -C identifiers or field names that are keywords in Go can be -accessed by prefixing them with an underscore: if x points at -a C struct with a field named "type", x._type accesses the field. +Alternatively, CFLAGS and LDFLAGS may be obtained via the pkg-config +tool using a '#cgo pkg-config:' directive followed by the package names. +For example: + + // #cgo pkg-config: png cairo + // #include <png.h> + import "C" + +Within the Go file, C identifiers or field names that are keywords in Go +can be accessed by prefixing them with an underscore: if x points at a C +struct with a field named "type", x._type accesses the field. The standard C numeric types are available under the names C.char, C.schar (signed char), C.uchar (unsigned char), diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go index ae5ca2c7d..10411e94f 100644 --- a/src/cmd/cgo/gcc.go +++ b/src/cmd/cgo/gcc.go @@ -100,27 +100,81 @@ NextLine: fatalf("%s: bad #cgo option: %s", srcfile, fields[0]) } - if k != "CFLAGS" && k != "LDFLAGS" { - fatalf("%s: unsupported #cgo option %s", srcfile, k) - } - - v := strings.TrimSpace(fields[1]) - args, err := splitQuoted(v) + args, err := splitQuoted(fields[1]) if err != nil { - fatalf("%s: bad #cgo option %s: %s", srcfile, k, err.String()) + fatalf("%s: bad #cgo option %s: %s", srcfile, k, err) } - if oldv, ok := p.CgoFlags[k]; ok { - p.CgoFlags[k] = oldv + " " + v - } else { - p.CgoFlags[k] = v + for _, arg := range args { + if !safeName(arg) { + fatalf("%s: #cgo option %s is unsafe: %s", srcfile, k, arg) + } } - if k == "CFLAGS" { - p.GccOptions = append(p.GccOptions, args...) + + switch k { + + case "CFLAGS", "LDFLAGS": + p.addToFlag(k, args) + + case "pkg-config": + cflags, ldflags, err := pkgConfig(args) + if err != nil { + fatalf("%s: bad #cgo option %s: %s", srcfile, k, err) + } + p.addToFlag("CFLAGS", cflags) + p.addToFlag("LDFLAGS", ldflags) + + default: + fatalf("%s: unsupported #cgo option %s", srcfile, k) + } } f.Preamble = strings.Join(linesOut, "\n") } +// addToFlag appends args to flag. All flags are later written out onto the +// _cgo_flags file for the build system to use. +func (p *Package) addToFlag(flag string, args []string) { + if oldv, ok := p.CgoFlags[flag]; ok { + p.CgoFlags[flag] = oldv + " " + strings.Join(args, " ") + } else { + p.CgoFlags[flag] = strings.Join(args, " ") + } + if flag == "CFLAGS" { + // We'll also need these when preprocessing for dwarf information. + p.GccOptions = append(p.GccOptions, args...) + } +} + +// pkgConfig runs pkg-config and extracts --libs and --cflags information +// for packages. +func pkgConfig(packages []string) (cflags, ldflags []string, err os.Error) { + for _, name := range packages { + if len(name) == 0 || name[0] == '-' { + return nil, nil, os.NewError(fmt.Sprintf("invalid name: %q", name)) + } + } + + args := append([]string{"pkg-config", "--cflags"}, packages...) + stdout, stderr, ok := run(nil, args) + if !ok { + os.Stderr.Write(stderr) + return nil, nil, os.NewError("pkg-config failed") + } + cflags, err = splitQuoted(string(stdout)) + if err != nil { + return + } + + args = append([]string{"pkg-config", "--libs"}, packages...) + stdout, stderr, ok = run(nil, args) + if !ok { + os.Stderr.Write(stderr) + return nil, nil, os.NewError("pkg-config failed") + } + ldflags, err = splitQuoted(string(stdout)) + return +} + // splitQuoted splits the string s around each instance of one or more consecutive // white space characters while taking into account quotes and escaping, and // returns an array of substrings of s or an empty list if s contains only white space. @@ -182,6 +236,20 @@ func splitQuoted(s string) (r []string, err os.Error) { return args, err } +var safeBytes = []byte("+-.,/0123456789=ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz") + +func safeName(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + if c := s[i]; c < 0x80 && bytes.IndexByte(safeBytes, c) < 0 { + return false + } + } + return true +} + // Translate rewrites f.AST, the original Go input, to remove // references to the imported package C, replacing them with // references to the equivalent Go types, functions, and variables. @@ -592,11 +660,14 @@ func (p *Package) gccName() (ret string) { } // gccMachine returns the gcc -m flag to use, either "-m32" or "-m64". -func (p *Package) gccMachine() string { - if p.PtrSize == 8 { - return "-m64" +func (p *Package) gccMachine() []string { + switch runtime.GOARCH { + case "amd64": + return []string{"-m64"} + case "386": + return []string{"-m32"} } - return "-m32" + return nil } const gccTmp = "_obj/_cgo_.o" @@ -604,9 +675,8 @@ const gccTmp = "_obj/_cgo_.o" // gccCmd returns the gcc command line to use for compiling // the input. func (p *Package) gccCmd() []string { - return []string{ + c := []string{ p.gccName(), - p.gccMachine(), "-Wall", // many warnings "-Werror", // warnings are errors "-o" + gccTmp, // write object to tmp @@ -614,15 +684,18 @@ func (p *Package) gccCmd() []string { "-fno-eliminate-unused-debug-types", // gets rid of e.g. untyped enum otherwise "-c", // do not link "-xc", // input language is C - "-", // read input from standard input } + c = append(c, p.GccOptions...) + c = append(c, p.gccMachine()...) + c = append(c, "-") //read input from standard input + return c } // gccDebug runs gcc -gdwarf-2 over the C program stdin and // returns the corresponding DWARF data and any messages // printed to standard error. func (p *Package) gccDebug(stdin []byte) *dwarf.Data { - runGcc(stdin, append(p.gccCmd(), p.GccOptions...)) + runGcc(stdin, p.gccCmd()) // Try to parse f as ELF and Mach-O and hope one works. var f interface { @@ -649,8 +722,9 @@ func (p *Package) gccDebug(stdin []byte) *dwarf.Data { // #defines that gcc encountered while processing the input // and its included files. func (p *Package) gccDefines(stdin []byte) string { - base := []string{p.gccName(), p.gccMachine(), "-E", "-dM", "-xc", "-"} - stdout, _ := runGcc(stdin, append(base, p.GccOptions...)) + base := []string{p.gccName(), "-E", "-dM", "-xc"} + base = append(base, p.gccMachine()...) + stdout, _ := runGcc(stdin, append(append(base, p.GccOptions...), "-")) return stdout } @@ -659,7 +733,7 @@ func (p *Package) gccDefines(stdin []byte) string { // gcc to fail. func (p *Package) gccErrors(stdin []byte) string { // TODO(rsc): require failure - args := append(p.gccCmd(), p.GccOptions...) + args := p.gccCmd() if *debugGcc { fmt.Fprintf(os.Stderr, "$ %s <<EOF\n", strings.Join(args, " ")) os.Stderr.Write(stdin) diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go index bc031cc58..dbc7bcf69 100644 --- a/src/cmd/cgo/out.go +++ b/src/cmd/cgo/out.go @@ -142,7 +142,7 @@ func dynimport(obj string) { if f, err := pe.Open(obj); err == nil { sym, err := f.ImportedSymbols() if err != nil { - fatalf("cannot load imported symbols from PE file %s: v", obj, err) + fatalf("cannot load imported symbols from PE file %s: %v", obj, err) } for _, s := range sym { ss := strings.Split(s, ":", -1) @@ -331,7 +331,11 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) { fmt.Fprintf(fgcc, "\tint e;\n") // assuming 32 bit (see comment above structType) fmt.Fprintf(fgcc, "\terrno = 0;\n") } - fmt.Fprintf(fgcc, "\t%s *a = v;\n", ctype) + // We're trying to write a gcc struct that matches 6c/8c/5c's layout. + // Use packed attribute to force no padding in this struct in case + // gcc has different packing requirements. For example, + // on 386 Windows, gcc wants to 8-align int64s, but 8c does not. + fmt.Fprintf(fgcc, "\t%s __attribute__((__packed__)) *a = v;\n", ctype) fmt.Fprintf(fgcc, "\t") if t := n.FuncType.Result; t != nil { fmt.Fprintf(fgcc, "a->r = ") @@ -370,7 +374,9 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) { fn := exp.Func // Construct a gcc struct matching the 6c argument and - // result frame. + // result frame. The gcc struct will be compiled with + // __attribute__((packed)) so all padding must be accounted + // for explicitly. ctype := "struct {\n" off := int64(0) npad := 0 @@ -458,7 +464,7 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) { fmt.Fprintf(fgcc, "extern _cgoexp%s_%s(void *, int);\n", cPrefix, exp.ExpName) fmt.Fprintf(fgcc, "\n%s\n", s) fmt.Fprintf(fgcc, "{\n") - fmt.Fprintf(fgcc, "\t%s a;\n", ctype) + fmt.Fprintf(fgcc, "\t%s __attribute__((packed)) a;\n", ctype) if gccResult != "void" && (len(fntype.Results.List) > 1 || len(fntype.Results.List[0].Names) > 1) { fmt.Fprintf(fgcc, "\t%s r;\n", gccResult) } diff --git a/src/cmd/gc/Makefile b/src/cmd/gc/Makefile index dbfd86474..286618ec1 100644 --- a/src/cmd/gc/Makefile +++ b/src/cmd/gc/Makefile @@ -49,7 +49,7 @@ include ../../Make.clib install: $(LIB) y1.tab.c: y.tab.c # make yystate global, yytname mutable - cat y.tab.c | sed '/ int yystate;/d; s/int yychar;/int yychar, yystate;/; s/static const char \*const yytname/const char *yytname/' >y1.tab.c + cat y.tab.c | sed '/ int yystate;/d; s/int yychar;/int yychar, yystate;/; s/static const char \*const yytname/const char *yytname/; s/char const \*yymsgp/char *yymsgp/' >y1.tab.c yerr.h: bisonerrors go.errors y.tab.h # y.tab.h rule generates y.output too awk -f bisonerrors y.output go.errors >yerr.h diff --git a/src/cmd/gc/align.c b/src/cmd/gc/align.c index a8454bf13..7fcac4833 100644 --- a/src/cmd/gc/align.c +++ b/src/cmd/gc/align.c @@ -234,9 +234,11 @@ dowidth(Type *t) if(t->bound > cap) yyerror("type %lT larger than address space", t); w = t->bound * t->type->width; - if(w == 0) - w = 1; t->align = t->type->align; + if(w == 0) { + w = 1; + t->align = 1; + } } else if(t->bound == -1) { w = sizeof_Array; @@ -253,10 +255,10 @@ dowidth(Type *t) if(t->funarg) fatal("dowidth fn struct %T", t); w = widstruct(t, 0, 1); - if(w == 0) + if(w == 0) { w = 1; - //if(t->align < widthptr) - // warn("align %d: %T\n", t->align, t); + t->align = 1; + } break; case TFUNC: diff --git a/src/cmd/gc/builtin.c.boot b/src/cmd/gc/builtin.c.boot index bdbca7f78..c9bf501d1 100644 --- a/src/cmd/gc/builtin.c.boot +++ b/src/cmd/gc/builtin.c.boot @@ -1,5 +1,6 @@ char *runtimeimport = "package runtime\n" + "import runtime \"runtime\"\n" "func \"\".new (? int32) *any\n" "func \"\".panicindex ()\n" "func \"\".panicslice ()\n" @@ -21,7 +22,6 @@ char *runtimeimport = "func \"\".printsp ()\n" "func \"\".goprintf ()\n" "func \"\".concatstring ()\n" - "func \"\".append ()\n" "func \"\".appendslice (typ *uint8, x any, y []any) any\n" "func \"\".cmpstring (? string, ? string) int\n" "func \"\".slicestring (? string, ? int, ? int) string\n" @@ -81,6 +81,7 @@ char *runtimeimport = "func \"\".selectgo (sel *uint8)\n" "func \"\".block ()\n" "func \"\".makeslice (typ *uint8, nel int64, cap int64) []any\n" + "func \"\".growslice (typ *uint8, old []any, cap int64) []any\n" "func \"\".sliceslice1 (old []any, lb uint64, width uint64) []any\n" "func \"\".sliceslice (old []any, lb uint64, hb uint64, width uint64) []any\n" "func \"\".slicearray (old *any, nel uint64, lb uint64, hb uint64, width uint64) []any\n" @@ -98,6 +99,7 @@ char *runtimeimport = "$$\n"; char *unsafeimport = "package unsafe\n" + "import runtime \"runtime\"\n" "type \"\".Pointer uintptr\n" "func \"\".Offsetof (? any) int\n" "func \"\".Sizeof (? any) int\n" diff --git a/src/cmd/gc/closure.c b/src/cmd/gc/closure.c index eb7014366..906dadbc9 100644 --- a/src/cmd/gc/closure.c +++ b/src/cmd/gc/closure.c @@ -75,7 +75,7 @@ closurebody(NodeList *body) } void -typecheckclosure(Node *func) +typecheckclosure(Node *func, int top) { Node *oldfn; NodeList *l; @@ -106,6 +106,10 @@ typecheckclosure(Node *func) v->op = 0; continue; } + // For a closure that is called in place, but not + // inside a go statement, avoid moving variables to the heap. + if ((top & (Ecall|Eproc)) == Ecall) + v->heapaddr->etype = 1; typecheck(&v->heapaddr, Erv); func->enter = list(func->enter, v->heapaddr); v->heapaddr = N; @@ -199,3 +203,41 @@ walkclosure(Node *func, NodeList **init) walkexpr(&call, init); return call; } + +// Special case for closures that get called in place. +// Optimize runtime.closure(X, __func__xxxx_, .... ) away +// to __func__xxxx_(Y ....). +// On entry, expect n->op == OCALL, n->left->op == OCLOSURE. +void +walkcallclosure(Node *n, NodeList **init) +{ + Node *z; + NodeList *ll, *cargs; + + walkexpr(&n->left, init); + cargs = n->left // FUNC runtime.closure + ->list // arguments + ->next // skip first + ->next; // skip second + + n->left = n->left // FUNC runtime.closure + ->list // arguments + ->next // skip first + ->n // AS (to indreg) + ->right; // argument == the generated function + + // New arg list for n. First the closure-args, stolen from + // runtime.closure's 3rd and following, + ll = nil; + for (; cargs; cargs = cargs->next) + ll = list(ll, cargs->n->right); // cargs->n is the OAS(INDREG, arg) + + // then an extra zero, to fill the dummy return pointer slot, + z = nod(OXXX, N, N); + nodconst(z, types[TUINTPTR], 0); + z->typecheck = 1; + ll = list(ll, z); + + // and finally the original parameter list. + n->list = concat(ll, n->list); +} diff --git a/src/cmd/gc/const.c b/src/cmd/gc/const.c index a36ec68c0..8fe9072b2 100644 --- a/src/cmd/gc/const.c +++ b/src/cmd/gc/const.c @@ -103,6 +103,8 @@ convlit1(Node **np, Type *t, int explicit) case ORSH: convlit1(&n->left, t, explicit && isideal(n->left->type)); t = n->left->type; + if(t != T && t->etype == TIDEAL && n->val.ctype != CTINT) + n->val = toint(n->val); if(t != T && !isint[t->etype]) { yyerror("invalid operation: %#N (shift of type %T)", n, t); t = T; @@ -514,6 +516,8 @@ evconst(Node *n) n->right = nr; if(nr->type && (issigned[nr->type->etype] || !isint[nr->type->etype])) goto illegal; + nl->val = toint(nl->val); + nr->val = toint(nr->val); break; } diff --git a/src/cmd/gc/dcl.c b/src/cmd/gc/dcl.c index 8e790ef08..335d056a0 100644 --- a/src/cmd/gc/dcl.c +++ b/src/cmd/gc/dcl.c @@ -188,6 +188,7 @@ declare(Node *n, int ctxt) else if(n->op == ONAME) gen = ++vargen; pushdcl(s); + n->curfn = curfn; } if(ctxt == PAUTO) n->xoffset = BADWIDTH; @@ -437,20 +438,6 @@ newtype(Sym *s) return t; } -/* - * type check top level declarations - */ -void -dclchecks(void) -{ - NodeList *l; - - for(l=externdcl; l; l=l->next) { - if(l->n->op != ONAME) - continue; - typecheck(&l->n, Erv); - } -} /* * := declarations @@ -524,6 +511,30 @@ colas(NodeList *left, NodeList *right) } /* + * declare the arguments in an + * interface field declaration. + */ +void +ifacedcl(Node *n) +{ + if(n->op != ODCLFIELD || n->right == N) + fatal("ifacedcl"); + + dclcontext = PAUTO; + markdcl(); + funcdepth++; + n->outer = curfn; + curfn = n; + funcargs(n->right); + + // funcbody is normally called after the parser has + // seen the body of a function but since an interface + // field declaration does not have a body, we must + // call it now to pop the current declaration context. + funcbody(n); +} + +/* * declare the function proper * and declare the arguments. * called in extern-declaration context @@ -1226,9 +1237,6 @@ funccompile(Node *n, int isclosure) if(curfn) fatal("funccompile %S inside %S", n->nname->sym, curfn->nname->sym); - curfn = n; - typechecklist(n->nbody, Etop); - curfn = nil; stksize = 0; dclcontext = PAUTO; diff --git a/src/cmd/gc/gen.c b/src/cmd/gc/gen.c index 857fcbef9..ad7b65b30 100644 --- a/src/cmd/gc/gen.c +++ b/src/cmd/gc/gen.c @@ -51,6 +51,8 @@ allocparams(void) } if(n->op != ONAME || n->class != PAUTO) continue; + if (n->xoffset != BADWIDTH) + continue; if(n->type == T) continue; dowidth(n->type); @@ -59,6 +61,8 @@ allocparams(void) fatal("bad width"); stksize += w; stksize = rnd(stksize, n->type->align); + if(thechar == '5') + stksize = rnd(stksize, widthptr); n->xoffset = -stksize; } lineno = lno; @@ -668,14 +672,18 @@ dotoffset(Node *n, int *oary, Node **nn) * make a new off the books */ void -tempname(Node *n, Type *t) +tempname(Node *nn, Type *t) { + Node *n; Sym *s; uint32 w; if(stksize < 0) fatal("tempname not during code generation"); + if (curfn == N) + fatal("no curfn for tempname"); + if(t == T) { yyerror("tempname called with nil type"); t = types[TINT32]; @@ -686,19 +694,25 @@ tempname(Node *n, Type *t) snprint(namebuf, sizeof(namebuf), "autotmp_%.4d", statuniqgen); statuniqgen++; s = lookup(namebuf); - memset(n, 0, sizeof(*n)); - n->op = ONAME; + n = nod(ONAME, N, N); n->sym = s; n->type = t; n->class = PAUTO; n->addable = 1; n->ullman = 1; n->noescape = 1; + n->curfn = curfn; + curfn->dcl = list(curfn->dcl, n); dowidth(t); w = t->width; stksize += w; stksize = rnd(stksize, t->align); + if(thechar == '5') + stksize = rnd(stksize, widthptr); n->xoffset = -stksize; - n->pun = anyregalloc(); + + // print("\ttmpname (%d): %N\n", stksize, n); + + *nn = *n; } diff --git a/src/cmd/gc/go.h b/src/cmd/gc/go.h index 4b4e8fcd1..c16903e77 100644 --- a/src/cmd/gc/go.h +++ b/src/cmd/gc/go.h @@ -225,6 +225,7 @@ struct Node Type* realtype; // as determined by typecheck NodeList* list; NodeList* rlist; + Node* orig; // original form, for printing, and tracking copies of ONAMEs // for-body NodeList* ninit; @@ -253,6 +254,7 @@ struct Node Node* ntype; Node* defn; Node* pack; // real package for import . names + Node* curfn; // function for local variables // ONAME func param with PHEAP Node* heapaddr; // temp holding heap address of param @@ -271,6 +273,7 @@ struct Node int32 lineno; int32 endlineno; vlong xoffset; + int32 stkdelta; // offset added by stack frame compaction phase. int32 ostk; int32 iota; }; @@ -516,15 +519,16 @@ enum enum { - Etop = 1<<1, // evaluated at statement level - Erv = 1<<2, // evaluated in value context + Etop = 1<<1, // evaluated at statement level + Erv = 1<<2, // evaluated in value context Etype = 1<<3, - Ecall = 1<<4, // call-only expressions are ok + Ecall = 1<<4, // call-only expressions are ok Efnstruct = 1<<5, // multivalue function returns are ok Eiota = 1<<6, // iota is ok Easgn = 1<<7, // assigning to expression Eindir = 1<<8, // indirecting through expression Eaddr = 1<<9, // taking address of expression + Eproc = 1<<10, // inside a go statement }; #define BITS 5 @@ -544,6 +548,7 @@ struct Var vlong offset; Sym* sym; Sym* gotype; + Node* node; int width; char name; char etype; @@ -814,8 +819,9 @@ int bset(Bits a, uint n); */ Node* closurebody(NodeList *body); void closurehdr(Node *ntype); -void typecheckclosure(Node *func); +void typecheckclosure(Node *func, int top); Node* walkclosure(Node *func, NodeList **init); +void walkcallclosure(Node *n, NodeList **init); /* * const.c @@ -858,7 +864,6 @@ NodeList* checkarglist(NodeList *all, int input); Node* colas(NodeList *left, NodeList *right); void colasdefn(NodeList *left, Node *defn); NodeList* constiter(NodeList *vl, Node *t, NodeList *cl); -void dclchecks(void); Node* dclname(Sym *s); void declare(Node *n, int ctxt); Type* dostruct(NodeList *l, int et); @@ -869,6 +874,7 @@ void funcbody(Node *n); void funccompile(Node *n, int isclosure); void funchdr(Node *n); Type* functype(Node *this, NodeList *in, NodeList *out); +void ifacedcl(Node *n); int isifacemethod(Type *f); void markdcl(void); Node* methodname(Node *n, Type *t); @@ -1103,6 +1109,7 @@ int istype(Type *t, int et); void linehist(char *file, int32 off, int relative); NodeList* list(NodeList *l, Node *n); NodeList* list1(Node *n); +void listsort(NodeList**, int(*f)(Node*, Node*)); Node* liststmt(NodeList *l); NodeList* listtreecopy(NodeList *l); Sym* lookup(char *name); @@ -1161,6 +1168,11 @@ int exportassignok(Type *t, char *desc); int islvalue(Node *n); Node* typecheck(Node **np, int top); void typechecklist(NodeList *l, int top); +Node* typecheckdef(Node *n); +void resumetypecopy(void); +void copytype(Node *n, Type *t); +void defertypecopy(Node *n, Type *t); +void queuemethod(Node *n); /* * unsafe.c @@ -1172,15 +1184,10 @@ Node* unsafenmagic(Node *n); */ Node* callnew(Type *t); Node* chanfn(char *name, int n, Type *t); -void copytype(Node *n, Type *t); -void defertypecopy(Node *n, Type *t); Node* mkcall(char *name, Type *t, NodeList **init, ...); Node* mkcall1(Node *fn, Type *t, NodeList **init, ...); -void queuemethod(Node *n); -void resumetypecopy(void); int vmatch1(Node *l, Node *r); void walk(Node *fn); -Node* walkdef(Node *n); void walkexpr(Node **np, NodeList **init); void walkexprlist(NodeList *l, NodeList **init); void walkexprlistsafe(NodeList *l, NodeList **init); @@ -1208,6 +1215,7 @@ EXTERN Prog* continpc; EXTERN Prog* breakpc; EXTERN Prog* pc; EXTERN Prog* firstpc; +EXTERN Prog* retpc; EXTERN Node* nodfp; @@ -1221,6 +1229,7 @@ void cgen_callinter(Node *n, Node *res, int proc); void cgen_ret(Node *n); void clearfat(Node *n); void compile(Node*); +void defframe(Prog*); int dgostringptr(Sym*, int off, char *str); int dgostrlitptr(Sym*, int off, Strlit*); int dstringptr(Sym *s, int off, char *str); diff --git a/src/cmd/gc/go.y b/src/cmd/gc/go.y index 7adfd002a..1278c2586 100644 --- a/src/cmd/gc/go.y +++ b/src/cmd/gc/go.y @@ -1380,6 +1380,7 @@ interfacedcl: new_name indcl { $$ = nod(ODCLFIELD, $1, $2); + ifacedcl($$); } | packname { @@ -1791,24 +1792,12 @@ hidden_opt_sym: } hidden_dcl: - hidden_opt_sym hidden_type + hidden_opt_sym hidden_type hidden_tag { $$ = nod(ODCLFIELD, $1, typenod($2)); + $$->val = $3; } -| hidden_opt_sym LDDD - { - Type *t; - - yyerror("invalid variadic function type in import - recompile import"); - - t = typ(TARRAY); - t->bound = -1; - t->type = typ(TINTER); - $$ = nod(ODCLFIELD, $1, typenod(t)); - $$->isddd = 1; - } - -| hidden_opt_sym LDDD hidden_type +| hidden_opt_sym LDDD hidden_type hidden_tag { Type *t; @@ -1817,6 +1806,7 @@ hidden_dcl: t->type = $3; $$ = nod(ODCLFIELD, $1, typenod(t)); $$->isddd = 1; + $$->val = $4; } hidden_structdcl: diff --git a/src/cmd/gc/init.c b/src/cmd/gc/init.c index af4eb0336..8818db08c 100644 --- a/src/cmd/gc/init.c +++ b/src/cmd/gc/init.c @@ -182,11 +182,14 @@ fninit(NodeList *n) // (11) a = nod(ORETURN, N, N); r = list(r, a); - exportsym(fn->nname); fn->nbody = r; funcbody(fn); + + curfn = fn; typecheck(&fn, Etop); + typechecklist(r, Etop); + curfn = nil; funccompile(fn, 0); } diff --git a/src/cmd/gc/lex.c b/src/cmd/gc/lex.c index 04dd0d5b9..88acb60af 100644 --- a/src/cmd/gc/lex.c +++ b/src/cmd/gc/lex.c @@ -5,7 +5,7 @@ #define EXTERN #include "go.h" #include "y.tab.h" -#include <ar.h> +#include <ar.h> #undef getc #undef ungetc @@ -235,13 +235,14 @@ main(int argc, char *argv[]) if(debug['f']) frame(1); - // Process top-level declarations in three phases. + // Process top-level declarations in four phases. // Phase 1: const, type, and names and types of funcs. // This will gather all the information about types // and methods but doesn't depend on any of it. // Phase 2: Variable assignments. // To check interface assignments, depends on phase 1. - // Phase 3: Function bodies. + // Phase 3: Type check function bodies. + // Phase 4: Compile function bodies. defercheckwidth(); for(l=xtop; l; l=l->next) if(l->n->op != ODCL && l->n->op != OAS) @@ -251,18 +252,31 @@ main(int argc, char *argv[]) typecheck(&l->n, Etop); resumetypecopy(); resumecheckwidth(); + + for(l=xtop; l; l=l->next) + if(l->n->op == ODCLFUNC) { + curfn = l->n; + typechecklist(l->n->nbody, Etop); + } + curfn = nil; + for(l=xtop; l; l=l->next) if(l->n->op == ODCLFUNC) funccompile(l->n, 0); + if(nerrors == 0) fninit(xtop); + while(closures) { l = closures; closures = nil; for(; l; l=l->next) funccompile(l->n, 1); } - dclchecks(); + + for(l=externdcl; l; l=l->next) + if(l->n->op == ONAME) + typecheck(&l->n, Erv); if(nerrors) errorexit(); diff --git a/src/cmd/gc/obj.c b/src/cmd/gc/obj.c index 9f4b7b318..f34fc76c8 100644 --- a/src/cmd/gc/obj.c +++ b/src/cmd/gc/obj.c @@ -284,6 +284,7 @@ stringsym(char *s, int len) off = dsname(sym, off, s+n, m); } off = duint8(sym, off, 0); // terminating NUL for runtime + off = (off+widthptr-1)&~(widthptr-1); // round to pointer alignment ggloblsym(sym, off, 1); text(); diff --git a/src/cmd/gc/pgen.c b/src/cmd/gc/pgen.c new file mode 100644 index 000000000..ab6186697 --- /dev/null +++ b/src/cmd/gc/pgen.c @@ -0,0 +1,224 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "gg.h" +#include "opt.h" + +static void compactframe(Prog* p); + +void +compile(Node *fn) +{ + Plist *pl; + Node nod1, *n; + Prog *ptxt; + int32 lno; + Type *t; + Iter save; + vlong oldstksize; + + if(newproc == N) { + newproc = sysfunc("newproc"); + deferproc = sysfunc("deferproc"); + deferreturn = sysfunc("deferreturn"); + panicindex = sysfunc("panicindex"); + panicslice = sysfunc("panicslice"); + throwreturn = sysfunc("throwreturn"); + } + + if(fn->nbody == nil) + return; + + // set up domain for labels + clearlabels(); + + lno = setlineno(fn); + + curfn = fn; + dowidth(curfn->type); + + if(curfn->type->outnamed) { + // add clearing of the output parameters + t = structfirst(&save, getoutarg(curfn->type)); + while(t != T) { + if(t->nname != N) { + n = nod(OAS, t->nname, N); + typecheck(&n, Etop); + curfn->nbody = concat(list1(n), curfn->nbody); + } + t = structnext(&save); + } + } + + hasdefer = 0; + walk(curfn); + if(nerrors != 0 || isblank(curfn->nname)) + goto ret; + + allocparams(); + + continpc = P; + breakpc = P; + + pl = newplist(); + pl->name = curfn->nname; + + setlineno(curfn); + + nodconst(&nod1, types[TINT32], 0); + ptxt = gins(ATEXT, curfn->nname, &nod1); + afunclit(&ptxt->from); + + ginit(); + genlist(curfn->enter); + + retpc = nil; + if(hasdefer || curfn->exit) { + Prog *p1; + + p1 = gjmp(nil); + retpc = gjmp(nil); + patch(p1, pc); + } + + genlist(curfn->nbody); + gclean(); + checklabels(); + if(nerrors != 0) + goto ret; + if(curfn->endlineno) + lineno = curfn->endlineno; + + if(curfn->type->outtuple != 0) + ginscall(throwreturn, 0); + + if(retpc) + patch(retpc, pc); + ginit(); + if(hasdefer) + ginscall(deferreturn, 0); + if(curfn->exit) + genlist(curfn->exit); + gclean(); + if(nerrors != 0) + goto ret; + pc->as = ARET; // overwrite AEND + pc->lineno = lineno; + + if(!debug['N'] || debug['R'] || debug['P']) { + regopt(ptxt); + } + + oldstksize = stksize; + if(thechar != '5') + compactframe(ptxt); + if(0) + print("compactframe: %ld to %ld\n", oldstksize, stksize); + + defframe(ptxt); + + if(0) + frame(0); + +ret: + lineno = lno; +} + + +// Sort the list of stack variables. autos after anything else, +// within autos, unused after used, and within used on reverse alignment. +// non-autos sort on offset. +static int +cmpstackvar(Node *a, Node *b) +{ + if (a->class != b->class) + return (a->class == PAUTO) ? 1 : -1; + if (a->class != PAUTO) + return a->xoffset - b->xoffset; + if ((a->used == 0) != (b->used == 0)) + return b->used - a->used; + return b->type->align - a->type->align; + +} + +static void +compactframe(Prog* ptxt) +{ + NodeList *ll; + Node* n; + Prog *p; + uint32 w; + + if (stksize == 0) + return; + + // Mark the PAUTO's unused. + for(ll=curfn->dcl; ll != nil; ll=ll->next) + if (ll->n->class == PAUTO && ll->n->op == ONAME) + ll->n->used = 0; + + // Sweep the prog list to mark any used nodes. + for (p = ptxt; p; p = p->link) { + if (p->from.type == D_AUTO && p->from.node) + p->from.node->used++; + + if (p->to.type == D_AUTO && p->to.node) + p->to.node->used++; + } + + listsort(&curfn->dcl, cmpstackvar); + + // Unused autos are at the end, chop 'em off. + ll = curfn->dcl; + n = ll->n; + if (n->class == PAUTO && n->op == ONAME && !n->used) { + curfn->dcl = nil; + stksize = 0; + return; + } + + for(ll = curfn->dcl; ll->next != nil; ll=ll->next) { + n = ll->next->n; + if (n->class == PAUTO && n->op == ONAME && !n->used) { + ll->next = nil; + curfn->dcl->end = ll; + break; + } + } + + // Reassign stack offsets of the locals that are still there. + stksize = 0; + for(ll = curfn->dcl; ll != nil; ll=ll->next) { + n = ll->n; + // TODO find out where the literal autos come from + if (n->class != PAUTO || n->op != ONAME) + continue; + + w = n->type->width; + if((w >= MAXWIDTH) || (w < 1)) + fatal("bad width"); + stksize += w; + stksize = rnd(stksize, n->type->align); + if(thechar == '5') + stksize = rnd(stksize, widthptr); + n->stkdelta = -stksize - n->xoffset; + } + + // Fixup instructions. + for (p = ptxt; p; p = p->link) { + if (p->from.type == D_AUTO && p->from.node) + p->from.offset += p->from.node->stkdelta; + + if (p->to.type == D_AUTO && p->to.node) + p->to.offset += p->to.node->stkdelta; + } + + // The debug information needs accurate offsets on the symbols. + for(ll = curfn->dcl ;ll != nil; ll=ll->next) { + if (ll->n->class != PAUTO || ll->n->op != ONAME) + continue; + ll->n->xoffset += ll->n->stkdelta; + ll->n->stkdelta = 0; + } +} diff --git a/src/cmd/gc/print.c b/src/cmd/gc/print.c index e03a14080..e88e0f844 100644 --- a/src/cmd/gc/print.c +++ b/src/cmd/gc/print.c @@ -134,6 +134,10 @@ exprfmt(Fmt *f, Node *n, int prec) fmtprint(f, "(node %O)", n->op); break; + case OREGISTER: + fmtprint(f, "%R", n->val.u.reg); + break; + case OLITERAL: if(n->sym != S) { fmtprint(f, "%S", n->sym); diff --git a/src/cmd/gc/runtime.go b/src/cmd/gc/runtime.go index 35d11eca9..00fc720b8 100644 --- a/src/cmd/gc/runtime.go +++ b/src/cmd/gc/runtime.go @@ -110,6 +110,7 @@ func selectgo(sel *byte) func block() func makeslice(typ *byte, nel int64, cap int64) (ary []any) +func growslice(typ *byte, old []any, n int64) (ary []any) func sliceslice1(old []any, lb uint64, width uint64) (ary []any) func sliceslice(old []any, lb uint64, hb uint64, width uint64) (ary []any) func slicearray(old *any, nel uint64, lb uint64, hb uint64, width uint64) (ary []any) diff --git a/src/cmd/gc/sinit.c b/src/cmd/gc/sinit.c index be96a1477..eb7ef31ec 100644 --- a/src/cmd/gc/sinit.c +++ b/src/cmd/gc/sinit.c @@ -173,10 +173,10 @@ initfix(NodeList *l) /* * from here down is the walk analysis - * of composit literals. + * of composite literals. * most of the work is to generate * data statements for the constant - * part of the composit literal. + * part of the composite literal. */ static void structlit(int ctxt, int pass, Node *n, Node *var, NodeList **init); diff --git a/src/cmd/gc/subr.c b/src/cmd/gc/subr.c index bb2505694..49797f9df 100644 --- a/src/cmd/gc/subr.c +++ b/src/cmd/gc/subr.c @@ -105,7 +105,7 @@ hcrash(void) flusherrors(); if(outfile) unlink(outfile); - *(int*)0 = 0; + *(volatile int*)0 = 0; } } @@ -480,6 +480,7 @@ nod(int op, Node *nleft, Node *nright) n->right = nright; n->lineno = parserline(); n->xoffset = BADWIDTH; + n->orig = n; return n; } @@ -1031,10 +1032,21 @@ Econv(Fmt *fp) return fmtstrcpy(fp, etnames[et]); } +static const char* classnames[] = { + "Pxxx", + "PEXTERN", + "PAUTO", + "PPARAM", + "PPARAMOUT", + "PPARAMREF", + "PFUNC", +}; + int Jconv(Fmt *fp) { Node *n; + char *s; n = va_arg(fp->args, Node*); if(n->ullman != 0) @@ -1049,12 +1061,18 @@ Jconv(Fmt *fp) if(n->lineno != 0) fmtprint(fp, " l(%d)", n->lineno); - if(n->xoffset != 0) - fmtprint(fp, " x(%lld)", n->xoffset); - - if(n->class != 0) - fmtprint(fp, " class(%d)", n->class); + if(n->xoffset != BADWIDTH) + fmtprint(fp, " x(%lld%+d)", n->xoffset, n->stkdelta); + if(n->class != 0) { + s = ""; + if (n->class & PHEAP) s = ",heap"; + if ((n->class & ~PHEAP) < nelem(classnames)) + fmtprint(fp, " class(%s%s)", classnames[n->class&~PHEAP], s); + else + fmtprint(fp, " class(%d?%s)", n->class&~PHEAP, s); + } + if(n->colas != 0) fmtprint(fp, " colas(%d)", n->colas); @@ -1073,6 +1091,11 @@ Jconv(Fmt *fp) if(n->implicit != 0) fmtprint(fp, " implicit(%d)", n->implicit); + if(n->pun != 0) + fmtprint(fp, " pun(%d)", n->pun); + + if(n->used != 0) + fmtprint(fp, " used(%d)", n->used); return 0; } @@ -1141,7 +1164,7 @@ Tpretty(Fmt *fp, Type *t) Type *t1; Sym *s; - if(debug['r']) { + if(0 && debug['r']) { debug['r'] = 0; fmtprint(fp, "%T (orig=%T)", t, t->orig); debug['r'] = 1; @@ -1454,6 +1477,8 @@ Nconv(Fmt *fp) } if(fp->flags & FmtSharp) { + if(n->orig != N) + n = n->orig; exprfmt(fp, n, 0); goto out; } @@ -3107,7 +3132,7 @@ genwrapper(Type *rcvr, Type *method, Sym *newnam, int iface) Type *tpad; int isddd; - if(debug['r']) + if(0 && debug['r']) print("genwrapper rcvrtype=%T method=%T newnam=%S\n", rcvr, method, newnam); @@ -3161,11 +3186,14 @@ genwrapper(Type *rcvr, Type *method, Sym *newnam, int iface) fn->nbody = list1(n); } - if(debug['r']) + if(0 && debug['r']) dumplist("genwrapper body", fn->nbody); funcbody(fn); + curfn = fn; typecheck(&fn, Etop); + typechecklist(fn->nbody, Etop); + curfn = nil; funccompile(fn, 0); } @@ -3256,8 +3284,9 @@ implements(Type *t, Type *iface, Type **m, Type **samename, int *ptr) // the method does not exist for value types. rcvr = getthisx(tm->type)->type->type; if(isptr[rcvr->etype] && !isptr[t0->etype] && !followptr && !isifacemethod(tm->type)) { - if(debug['r']) + if(0 && debug['r']) yyerror("interface pointer mismatch"); + *m = im; *samename = nil; *ptr = 1; @@ -3330,6 +3359,64 @@ list(NodeList *l, Node *n) return concat(l, list1(n)); } +void +listsort(NodeList** l, int(*f)(Node*, Node*)) +{ + NodeList *l1, *l2, *le; + + if(*l == nil || (*l)->next == nil) + return; + + l1 = *l; + l2 = *l; + for(;;) { + l2 = l2->next; + if(l2 == nil) + break; + l2 = l2->next; + if(l2 == nil) + break; + l1 = l1->next; + } + + l2 = l1->next; + l1->next = nil; + l2->end = (*l)->end; + (*l)->end = l1; + + l1 = *l; + listsort(&l1, f); + listsort(&l2, f); + + if ((*f)(l1->n, l2->n) < 0) { + *l = l1; + } else { + *l = l2; + l2 = l1; + l1 = *l; + } + + // now l1 == *l; and l1 < l2 + + while ((l1 != nil) && (l2 != nil)) { + while ((l1->next != nil) && (*f)(l1->next->n, l2->n) < 0) + l1 = l1->next; + + // l1 is last one from l1 that is < l2 + le = l1->next; // le is the rest of l1, first one that is >= l2 + if (le != nil) + le->end = (*l)->end; + + (*l)->end = l1; // cut *l at l1 + *l = concat(*l, l2); // glue l2 to *l's tail + + l1 = l2; // l1 is the first element of *l that is < the new l2 + l2 = le; // ... because l2 now is the old tail of l1 + } + + *l = concat(*l, l2); // any remainder +} + NodeList* listtreecopy(NodeList *l) { diff --git a/src/cmd/gc/swt.c b/src/cmd/gc/swt.c index 6e8436c3c..c2968c44b 100644 --- a/src/cmd/gc/swt.c +++ b/src/cmd/gc/swt.c @@ -867,8 +867,11 @@ typecheckswitch(Node *n) case Etype: // type switch if(ll->n->op == OLITERAL && istype(ll->n->type, TNIL)) ; - else if(ll->n->op != OTYPE && ll->n->type != T) + else if(ll->n->op != OTYPE && ll->n->type != T) { yyerror("%#N is not a type", ll->n); + // reset to original type + ll->n = n->ntest->right; + } break; } } diff --git a/src/cmd/gc/typecheck.c b/src/cmd/gc/typecheck.c index c48bf7a29..04dc1a507 100644 --- a/src/cmd/gc/typecheck.c +++ b/src/cmd/gc/typecheck.c @@ -8,7 +8,6 @@ * evaluates compile time constants. * marks variables that escape the local frame. * rewrites n->op to be more specific in some cases. - * sets n->walk to walking function. */ #include "go.h" @@ -29,10 +28,12 @@ static void typecheckfunc(Node*); static void checklvalue(Node*, char*); static void checkassign(Node*); static void checkassignlist(NodeList*); -static void stringtoarraylit(Node**); -static Node* resolve(Node*); +static void stringtoarraylit(Node**); +static Node* resolve(Node*); static Type* getforwtype(Node*); +static NodeList* typecheckdefstack; + /* * resolve ONONAME to definition, if any. */ @@ -159,7 +160,7 @@ typecheck(Node **np, int top) if(n->op == OTYPE && (ft = getforwtype(n->ntype)) != T) defertypecopy(n, ft); - walkdef(n); + typecheckdef(n); n->realtype = n->type; if(n->op == ONONAME) goto error; @@ -780,7 +781,7 @@ reswitch: n = r; goto reswitch; } - typecheck(&n->left, Erv | Etype | Ecall); + typecheck(&n->left, Erv | Etype | Ecall |(top&Eproc)); l = n->left; if(l->op == ONAME && l->etype != 0) { if(n->isddd && l->etype != OAPPEND) @@ -794,7 +795,7 @@ reswitch: defaultlit(&n->left, T); l = n->left; if(l->op == OTYPE) { - if(n->isddd) + if(n->isddd || l->type->bound == -100) yyerror("invalid use of ... in type conversion", l); // pick off before type-checking arguments ok |= Erv; @@ -822,7 +823,13 @@ reswitch: case ODOTMETH: n->op = OCALLMETH; - typecheckaste(OCALL, n->left, 0, getthisx(t), list1(l->left), "method receiver"); + // typecheckaste was used here but there wasn't enough + // information further down the call chain to know if we + // were testing a method receiver for unexported fields. + // It isn't necessary, so just do a sanity check. + tp = getthisx(t)->type->type; + if(l->left == N || !eqtype(l->left->type, tp)) + fatal("method receiver"); break; default: @@ -894,12 +901,20 @@ reswitch: // might be constant switch(t->etype) { case TSTRING: - if(isconst(l, CTSTR)) - nodconst(n, types[TINT], l->val.u.sval->len); + if(isconst(l, CTSTR)) { + r = nod(OXXX, N, N); + nodconst(r, types[TINT], l->val.u.sval->len); + r->orig = n; + n = r; + } break; case TARRAY: - if(t->bound >= 0 && l->op == ONAME) - nodconst(n, types[TINT], t->bound); + if(t->bound >= 0 && l->op == ONAME) { + r = nod(OXXX, N, N); + nodconst(r, types[TINT], t->bound); + r->orig = n; + n = r; + } break; } n->type = types[TINT]; @@ -1013,9 +1028,9 @@ reswitch: // copy([]byte, string) if(isslice(n->left->type) && n->right->type->etype == TSTRING) { - if (n->left->type->type ==types[TUINT8]) - goto ret; - yyerror("arguments to copy have different element types: %lT and string", n->left->type); + if (n->left->type->type == types[TUINT8]) + goto ret; + yyerror("arguments to copy have different element types: %lT and string", n->left->type); goto error; } @@ -1203,7 +1218,7 @@ reswitch: case OCLOSURE: ok |= Erv; - typecheckclosure(n); + typecheckclosure(n, top); if(n->type == T) goto error; goto ret; @@ -1232,11 +1247,15 @@ reswitch: goto ret; case ODEFER: - case OPROC: ok |= Etop; typecheck(&n->left, Etop); goto ret; + case OPROC: + ok |= Etop; + typecheck(&n->left, Etop|Eproc); + goto ret; + case OFOR: ok |= Etop; typechecklist(n->ninit, Etop); @@ -1357,7 +1376,10 @@ ret: goto error; } if((top & Etop) && !(top & (Ecall|Erv|Etype)) && !(ok & Etop)) { - yyerror("%#N not used", n); + if(n->diag == 0) { + yyerror("%#N not used", n); + n->diag = 1; + } goto error; } @@ -2148,7 +2170,9 @@ addrescapes(Node *n) if(n->noescape) break; switch(n->class) { - case PAUTO: + case PPARAMREF: + addrescapes(n->defn); + break; case PPARAM: case PPARAMOUT: // if func param, need separate temporary @@ -2156,16 +2180,17 @@ addrescapes(Node *n) // the function type has already been checked // (we're in the function body) // so the param already has a valid xoffset. - if(n->class == PPARAM || n->class == PPARAMOUT) { - // expression to refer to stack copy - n->stackparam = nod(OPARAM, n, N); - n->stackparam->type = n->type; - n->stackparam->addable = 1; - if(n->xoffset == BADWIDTH) - fatal("addrescapes before param assignment"); - n->stackparam->xoffset = n->xoffset; - n->xoffset = 0; - } + + // expression to refer to stack copy + n->stackparam = nod(OPARAM, n, N); + n->stackparam->type = n->type; + n->stackparam->addable = 1; + if(n->xoffset == BADWIDTH) + fatal("addrescapes before param assignment"); + n->stackparam->xoffset = n->xoffset; + n->xoffset = 0; + // fallthrough + case PAUTO: n->class |= PHEAP; n->addable = 0; @@ -2178,7 +2203,9 @@ addrescapes(Node *n) snprint(buf, sizeof buf, "&%S", n->sym); n->heapaddr->sym = lookup(buf); n->heapaddr->class = PHEAP-1; // defer tempname to allocparams - curfn->dcl = list(curfn->dcl, n->heapaddr); + n->heapaddr->ullman = 1; + n->curfn->dcl = list(n->curfn->dcl, n->heapaddr); + break; } break; @@ -2497,3 +2524,294 @@ getforwtype(Node *n) } } } + +static int ntypecheckdeftype; +static NodeList *methodqueue; + +static void +domethod(Node *n) +{ + Node *nt; + + nt = n->type->nname; + typecheck(&nt, Etype); + if(nt->type == T) { + // type check failed; leave empty func + n->type->etype = TFUNC; + n->type->nod = N; + return; + } + *n->type = *nt->type; + n->type->nod = N; + checkwidth(n->type); +} + +typedef struct NodeTypeList NodeTypeList; +struct NodeTypeList { + Node *n; + Type *t; + NodeTypeList *next; +}; + +static NodeTypeList *dntq; +static NodeTypeList *dntend; + +void +defertypecopy(Node *n, Type *t) +{ + NodeTypeList *ntl; + + if(n == N || t == T) + return; + + ntl = mal(sizeof *ntl); + ntl->n = n; + ntl->t = t; + ntl->next = nil; + + if(dntq == nil) + dntq = ntl; + else + dntend->next = ntl; + + dntend = ntl; +} + +void +resumetypecopy(void) +{ + NodeTypeList *l; + + for(l=dntq; l; l=l->next) + copytype(l->n, l->t); +} + +void +copytype(Node *n, Type *t) +{ + *n->type = *t; + + t = n->type; + t->sym = n->sym; + t->local = n->local; + t->vargen = n->vargen; + t->siggen = 0; + t->method = nil; + t->nod = N; + t->printed = 0; + t->deferwidth = 0; +} + +static void +typecheckdeftype(Node *n) +{ + int maplineno, embedlineno, lno; + Type *t; + NodeList *l; + + ntypecheckdeftype++; + lno = lineno; + setlineno(n); + n->type->sym = n->sym; + n->typecheck = 1; + typecheck(&n->ntype, Etype); + if((t = n->ntype->type) == T) { + n->diag = 1; + goto ret; + } + if(n->type == T) { + n->diag = 1; + goto ret; + } + + maplineno = n->type->maplineno; + embedlineno = n->type->embedlineno; + + // copy new type and clear fields + // that don't come along. + // anything zeroed here must be zeroed in + // typedcl2 too. + copytype(n, t); + + // double-check use of type as map key. + if(maplineno) { + lineno = maplineno; + maptype(n->type, types[TBOOL]); + } + if(embedlineno) { + lineno = embedlineno; + if(isptr[t->etype]) + yyerror("embedded type cannot be a pointer"); + } + +ret: + lineno = lno; + + // if there are no type definitions going on, it's safe to + // try to resolve the method types for the interfaces + // we just read. + if(ntypecheckdeftype == 1) { + while((l = methodqueue) != nil) { + methodqueue = nil; + for(; l; l=l->next) + domethod(l->n); + } + } + ntypecheckdeftype--; +} + +void +queuemethod(Node *n) +{ + if(ntypecheckdeftype == 0) { + domethod(n); + return; + } + methodqueue = list(methodqueue, n); +} + +Node* +typecheckdef(Node *n) +{ + int lno; + Node *e; + Type *t; + NodeList *l; + + lno = lineno; + setlineno(n); + + if(n->op == ONONAME) { + if(!n->diag) { + n->diag = 1; + if(n->lineno != 0) + lineno = n->lineno; + yyerror("undefined: %S", n->sym); + } + return n; + } + + if(n->walkdef == 1) + return n; + + l = mal(sizeof *l); + l->n = n; + l->next = typecheckdefstack; + typecheckdefstack = l; + + if(n->walkdef == 2) { + flusherrors(); + print("typecheckdef loop:"); + for(l=typecheckdefstack; l; l=l->next) + print(" %S", l->n->sym); + print("\n"); + fatal("typecheckdef loop"); + } + n->walkdef = 2; + + if(n->type != T || n->sym == S) // builtin or no name + goto ret; + + switch(n->op) { + default: + fatal("typecheckdef %O", n->op); + + case OLITERAL: + if(n->ntype != N) { + typecheck(&n->ntype, Etype); + n->type = n->ntype->type; + n->ntype = N; + if(n->type == T) { + n->diag = 1; + goto ret; + } + } + e = n->defn; + n->defn = N; + if(e == N) { + lineno = n->lineno; + dump("typecheckdef nil defn", n); + yyerror("xxx"); + } + typecheck(&e, Erv | Eiota); + if(e->type != T && e->op != OLITERAL) { + yyerror("const initializer must be constant"); + goto ret; + } + if(isconst(e, CTNIL)) { + yyerror("const initializer cannot be nil"); + goto ret; + } + t = n->type; + if(t != T) { + if(!okforconst[t->etype]) { + yyerror("invalid constant type %T", t); + goto ret; + } + if(!isideal(e->type) && !eqtype(t, e->type)) { + yyerror("cannot use %+N as type %T in const initializer", e, t); + goto ret; + } + convlit(&e, t); + } + n->val = e->val; + n->type = e->type; + break; + + case ONAME: + if(n->ntype != N) { + typecheck(&n->ntype, Etype); + n->type = n->ntype->type; + if(n->type == T) { + n->diag = 1; + goto ret; + } + } + if(n->type != T) + break; + if(n->defn == N) { + if(n->etype != 0) // like OPRINTN + break; + if(nerrors > 0) { + // Can have undefined variables in x := foo + // that make x have an n->ndefn == nil. + // If there are other errors anyway, don't + // bother adding to the noise. + break; + } + fatal("var without type, init: %S", n->sym); + } + if(n->defn->op == ONAME) { + typecheck(&n->defn, Erv); + n->type = n->defn->type; + break; + } + typecheck(&n->defn, Etop); // fills in n->type + break; + + case OTYPE: + if(curfn) + defercheckwidth(); + n->walkdef = 1; + n->type = typ(TFORW); + n->type->sym = n->sym; + typecheckdeftype(n); + if(curfn) + resumecheckwidth(); + break; + + case OPACK: + // nothing to see here + break; + } + +ret: + if(typecheckdefstack->n != n) + fatal("typecheckdefstack mismatch"); + l = typecheckdefstack; + typecheckdefstack = l->next; + + lineno = lno; + n->walkdef = 1; + return n; +} diff --git a/src/cmd/gc/unsafe.c b/src/cmd/gc/unsafe.c index 33f375631..540994ddd 100644 --- a/src/cmd/gc/unsafe.c +++ b/src/cmd/gc/unsafe.c @@ -41,6 +41,7 @@ unsafenmagic(Node *nn) tr = r->type; if(tr == T) goto bad; + dowidth(tr); v = tr->width; goto yes; } diff --git a/src/cmd/gc/walk.c b/src/cmd/gc/walk.c index 278eef414..65a504bff 100644 --- a/src/cmd/gc/walk.c +++ b/src/cmd/gc/walk.c @@ -18,12 +18,11 @@ static NodeList* paramstoheap(Type **argin, int out); static NodeList* reorder1(NodeList*); static NodeList* reorder3(NodeList*); static Node* addstr(Node*, NodeList**); +static Node* appendslice(Node*, NodeList**); static Node* append(Node*, NodeList**); -static NodeList* walkdefstack; - // can this code branch reach the end -// without an undcontitional RETURN +// without an unconditional RETURN // this is hard, so it is conservative static int walkret(NodeList *l) @@ -65,6 +64,7 @@ walk(Node *fn) int lno; curfn = fn; + if(debug['W']) { snprint(s, sizeof(s), "\nbefore %S", curfn->nname->sym); dumplist(s, curfn->nbody); @@ -72,7 +72,7 @@ walk(Node *fn) if(curfn->type->outtuple) if(walkret(curfn->nbody)) yyerror("function ends without a return statement"); - typechecklist(curfn->nbody, Etop); + lno = lineno; for(l=fn->dcl; l; l=l->next) { n = l->n; @@ -98,296 +98,6 @@ walk(Node *fn) } } -static int nwalkdeftype; -static NodeList *methodqueue; - -static void -domethod(Node *n) -{ - Node *nt; - - nt = n->type->nname; - typecheck(&nt, Etype); - if(nt->type == T) { - // type check failed; leave empty func - n->type->etype = TFUNC; - n->type->nod = N; - return; - } - *n->type = *nt->type; - n->type->nod = N; - checkwidth(n->type); -} - -typedef struct NodeTypeList NodeTypeList; -struct NodeTypeList { - Node *n; - Type *t; - NodeTypeList *next; -}; - -static NodeTypeList *dntq; -static NodeTypeList *dntend; - -void -defertypecopy(Node *n, Type *t) -{ - NodeTypeList *ntl; - - if(n == N || t == T) - return; - - ntl = mal(sizeof *ntl); - ntl->n = n; - ntl->t = t; - ntl->next = nil; - - if(dntq == nil) - dntq = ntl; - else - dntend->next = ntl; - - dntend = ntl; -} - -void -resumetypecopy(void) -{ - NodeTypeList *l; - - for(l=dntq; l; l=l->next) - copytype(l->n, l->t); -} - -void -copytype(Node *n, Type *t) -{ - *n->type = *t; - - t = n->type; - t->sym = n->sym; - t->local = n->local; - t->vargen = n->vargen; - t->siggen = 0; - t->method = nil; - t->nod = N; - t->printed = 0; - t->deferwidth = 0; -} - -static void -walkdeftype(Node *n) -{ - int maplineno, embedlineno, lno; - Type *t; - NodeList *l; - - nwalkdeftype++; - lno = lineno; - setlineno(n); - n->type->sym = n->sym; - n->typecheck = 1; - typecheck(&n->ntype, Etype); - if((t = n->ntype->type) == T) { - n->diag = 1; - goto ret; - } - if(n->type == T) { - n->diag = 1; - goto ret; - } - - maplineno = n->type->maplineno; - embedlineno = n->type->embedlineno; - - // copy new type and clear fields - // that don't come along. - // anything zeroed here must be zeroed in - // typedcl2 too. - copytype(n, t); - - // double-check use of type as map key. - if(maplineno) { - lineno = maplineno; - maptype(n->type, types[TBOOL]); - } - if(embedlineno) { - lineno = embedlineno; - if(isptr[t->etype]) - yyerror("embedded type cannot be a pointer"); - } - -ret: - lineno = lno; - - // if there are no type definitions going on, it's safe to - // try to resolve the method types for the interfaces - // we just read. - if(nwalkdeftype == 1) { - while((l = methodqueue) != nil) { - methodqueue = nil; - for(; l; l=l->next) - domethod(l->n); - } - } - nwalkdeftype--; -} - -void -queuemethod(Node *n) -{ - if(nwalkdeftype == 0) { - domethod(n); - return; - } - methodqueue = list(methodqueue, n); -} - -Node* -walkdef(Node *n) -{ - int lno; - Node *e; - Type *t; - NodeList *l; - - lno = lineno; - setlineno(n); - - if(n->op == ONONAME) { - if(!n->diag) { - n->diag = 1; - if(n->lineno != 0) - lineno = n->lineno; - yyerror("undefined: %S", n->sym); - } - return n; - } - - if(n->walkdef == 1) - return n; - - l = mal(sizeof *l); - l->n = n; - l->next = walkdefstack; - walkdefstack = l; - - if(n->walkdef == 2) { - flusherrors(); - print("walkdef loop:"); - for(l=walkdefstack; l; l=l->next) - print(" %S", l->n->sym); - print("\n"); - fatal("walkdef loop"); - } - n->walkdef = 2; - - if(n->type != T || n->sym == S) // builtin or no name - goto ret; - - switch(n->op) { - default: - fatal("walkdef %O", n->op); - - case OLITERAL: - if(n->ntype != N) { - typecheck(&n->ntype, Etype); - n->type = n->ntype->type; - n->ntype = N; - if(n->type == T) { - n->diag = 1; - goto ret; - } - } - e = n->defn; - n->defn = N; - if(e == N) { - lineno = n->lineno; - dump("walkdef nil defn", n); - yyerror("xxx"); - } - typecheck(&e, Erv | Eiota); - if(e->type != T && e->op != OLITERAL) { - yyerror("const initializer must be constant"); - goto ret; - } - if(isconst(e, CTNIL)) { - yyerror("const initializer cannot be nil"); - goto ret; - } - t = n->type; - if(t != T) { - if(!okforconst[t->etype]) { - yyerror("invalid constant type %T", t); - goto ret; - } - if(!isideal(e->type) && !eqtype(t, e->type)) { - yyerror("cannot use %+N as type %T in const initializer", e, t); - goto ret; - } - convlit(&e, t); - } - n->val = e->val; - n->type = e->type; - break; - - case ONAME: - if(n->ntype != N) { - typecheck(&n->ntype, Etype); - n->type = n->ntype->type; - if(n->type == T) { - n->diag = 1; - goto ret; - } - } - if(n->type != T) - break; - if(n->defn == N) { - if(n->etype != 0) // like OPRINTN - break; - if(nerrors > 0) { - // Can have undefined variables in x := foo - // that make x have an n->ndefn == nil. - // If there are other errors anyway, don't - // bother adding to the noise. - break; - } - fatal("var without type, init: %S", n->sym); - } - if(n->defn->op == ONAME) { - typecheck(&n->defn, Erv); - n->type = n->defn->type; - break; - } - typecheck(&n->defn, Etop); // fills in n->type - break; - - case OTYPE: - if(curfn) - defercheckwidth(); - n->walkdef = 1; - n->type = typ(TFORW); - n->type->sym = n->sym; - walkdeftype(n); - if(curfn) - resumecheckwidth(); - break; - - case OPACK: - // nothing to see here - break; - } - -ret: - if(walkdefstack->n != n) - fatal("walkdefstack mismatch"); - l = walkdefstack; - walkdefstack = l->next; - - lineno = lno; - n->walkdef = 1; - return n; -} void walkstmtlist(NodeList *l) @@ -467,8 +177,10 @@ walkstmt(Node **np) case OPANIC: case OEMPTY: case ORECOVER: - if(n->typecheck == 0) + if(n->typecheck == 0) { + dump("missing typecheck:", n); fatal("missing typecheck"); + } init = n->ninit; n->ninit = nil; walkexpr(&n, &init); @@ -769,8 +481,15 @@ walkexpr(Node **np, NodeList **init) t = n->left->type; if(n->list && n->list->n->op == OAS) goto ret; - walkexpr(&n->left, init); + + if(n->left->op == OCLOSURE) { + walkcallclosure(n, init); + t = n->left->type; + } else + walkexpr(&n->left, init); + walkexprlist(n->list, init); + ll = ascompatte(n->op, n->isddd, getinarg(t), n->list, 0, init); n->list = reorder1(ll); if(isselect(n)) { @@ -805,18 +524,17 @@ walkexpr(Node **np, NodeList **init) n->ninit = nil; walkexpr(&n->left, init); n->left = safeexpr(n->left, init); + if(oaslit(n, init)) goto ret; + walkexpr(&n->right, init); - l = n->left; - r = n->right; - if(l == N || r == N) - goto ret; - r = ascompatee1(n->op, l, r, init); - if(r != N) { + if(n->left != N && n->right != N) { + r = convas(nod(OAS, n->left, n->right), init); r->dodata = n->dodata; n = r; } + goto ret; case OAS2: @@ -1134,6 +852,7 @@ walkexpr(Node **np, NodeList **init) case OINDEXMAP: if(n->etype == 1) goto ret; + t = n->left->type; n = mkcall1(mapfn("mapaccess1", t), t->type, init, n->left, n->right); goto ret; @@ -1188,6 +907,7 @@ walkexpr(Node **np, NodeList **init) // sliceslice(old []any, lb uint64, hb uint64, width uint64) (ary []any) // sliceslice1(old []any, lb uint64, width uint64) (ary []any) t = n->type; + et = n->etype; if(n->right->left == N) l = nodintconst(0); else @@ -1210,6 +930,7 @@ walkexpr(Node **np, NodeList **init) l, nodintconst(t->type->width)); } + n->etype = et; // preserve no-typecheck flag from OSLICE to the slice* call. goto ret; slicearray: @@ -1332,7 +1053,10 @@ walkexpr(Node **np, NodeList **init) goto ret; case OAPPEND: - n = append(n, init); + if(n->isddd) + n = appendslice(n, init); + else + n = append(n, init); goto ret; case OCOPY: @@ -1519,7 +1243,7 @@ ascompatee(int op, NodeList *nl, NodeList *nr, NodeList **init) static int fncall(Node *l, Type *rt) { - if(l->ullman >= UINF) + if(l->ullman >= UINF || l->op == OINDEXMAP) return 1; if(eqtype(l->type, rt)) return 0; @@ -1953,23 +1677,18 @@ callnew(Type *t) static Node* convas(Node *n, NodeList **init) { - Node *l, *r; Type *lt, *rt; if(n->op != OAS) fatal("convas: not OAS %O", n->op); - n->typecheck = 1; - lt = T; - rt = T; + n->typecheck = 1; - l = n->left; - r = n->right; - if(l == N || r == N) + if(n->left == N || n->right == N) goto out; - lt = l->type; - rt = r->type; + lt = n->left->type; + rt = n->right->type; if(lt == T || rt == T) goto out; @@ -1987,7 +1706,7 @@ convas(Node *n, NodeList **init) if(eqtype(lt, rt)) goto out; - n->right = assignconv(r, lt, "assignment"); + n->right = assignconv(n->right, lt, "assignment"); walkexpr(&n->right, init); out: @@ -2365,42 +2084,85 @@ addstr(Node *n, NodeList **init) } static Node* -append(Node *n, NodeList **init) +appendslice(Node *n, NodeList **init) { - int i, j; - Node *f, *r; - NodeList *in, *args; + Node *f; - if(n->isddd) { - f = syslook("appendslice", 1); - argtype(f, n->type); - argtype(f, n->type->type); - argtype(f, n->type); - r = mkcall1(f, n->type, init, typename(n->type), n->list->n, n->list->next->n); - return r; + f = syslook("appendslice", 1); + argtype(f, n->type); + argtype(f, n->type->type); + argtype(f, n->type); + return mkcall1(f, n->type, init, typename(n->type), n->list->n, n->list->next->n); +} + +// expand append(src, a [, b]* ) to +// +// init { +// s := src +// const argc = len(args) - 1 +// if cap(s) - len(s) < argc { +// s = growslice(s, argc) +// } +// n := len(s) +// s = s[:n+argc] +// s[n] = a +// s[n+1] = b +// ... +// } +// s +static Node* +append(Node *n, NodeList **init) +{ + NodeList *l, *a; + Node *nsrc, *ns, *nn, *na, *nx, *fn; + int argc; + + walkexprlistsafe(n->list, init); + + nsrc = n->list->n; + argc = count(n->list) - 1; + if (argc < 1) { + return nsrc; } - j = count(n->list) - 1; - f = syslook("append", 1); - f->type = T; - f->ntype = nod(OTFUNC, N, N); - in = list1(nod(ODCLFIELD, N, typenod(ptrto(types[TUINT8])))); // type - in = list(in, nod(ODCLFIELD, N, typenod(types[TINT]))); // count - in = list(in, nod(ODCLFIELD, N, typenod(n->type))); // slice - for(i=0; i<j; i++) - in = list(in, nod(ODCLFIELD, N, typenod(n->type->type))); - f->ntype->list = in; - f->ntype->rlist = list1(nod(ODCLFIELD, N, typenod(n->type))); - - args = list1(typename(n->type)); - args = list(args, nodintconst(j)); - args = concat(args, n->list); - - r = nod(OCALL, f, N); - r->list = args; - typecheck(&r, Erv); - walkexpr(&r, init); - r->type = n->type; + l = nil; - return r; + ns = nod(OXXX, N, N); // var s + tempname(ns, nsrc->type); + l = list(l, nod(OAS, ns, nsrc)); // s = src + + na = nodintconst(argc); // const argc + nx = nod(OIF, N, N); // if cap(s) - len(s) < argc + nx->ntest = nod(OLT, nod(OSUB, nod(OCAP, ns, N), nod(OLEN, ns, N)), na); + + fn = syslook("growslice", 1); // growslice(<type>, old []T, n int64) (ret []T) + argtype(fn, ns->type->type); // 1 old []any + argtype(fn, ns->type->type); // 2 ret []any + + nx->nbody = list1(nod(OAS, ns, mkcall1(fn, ns->type, &nx->ninit, + typename(ns->type), + ns, + conv(na, types[TINT64])))); + l = list(l, nx); + + nn = nod(OXXX, N, N); // var n + tempname(nn, types[TINT]); + l = list(l, nod(OAS, nn, nod(OLEN, ns, N))); // n = len(s) + + nx = nod(OSLICE, ns, nod(OKEY, N, nod(OADD, nn, na))); // ...s[:n+argc] + nx->etype = 1; // disable bounds check + l = list(l, nod(OAS, ns, nx)); // s = s[:n+argc] + + for (a = n->list->next; a != nil; a = a->next) { + nx = nod(OINDEX, ns, nn); // s[n] ... + nx->etype = 1; // disable bounds check + l = list(l, nod(OAS, nx, a->n)); // s[n] = arg + if (a->next != nil) + l = list(l, nod(OAS, nn, nod(OADD, nn, nodintconst(1)))); // n = n + 1 + } + + typechecklist(l, Etop); + walkstmtlist(l); + *init = concat(*init, l); + return ns; } diff --git a/src/cmd/godefs/a.h b/src/cmd/godefs/a.h index 03ab91f65..9b4957467 100644 --- a/src/cmd/godefs/a.h +++ b/src/cmd/godefs/a.h @@ -75,6 +75,7 @@ extern Const *con; extern int ncon; extern Type **typ; extern int ntyp; +extern int kindsize[]; // Language output typedef struct Lang Lang; diff --git a/src/cmd/godefs/main.c b/src/cmd/godefs/main.c index d4163421d..6a8630179 100644 --- a/src/cmd/godefs/main.c +++ b/src/cmd/godefs/main.c @@ -181,7 +181,7 @@ main(int argc, char **argv) char **av, *q, *r, *tofree, *name; char nambuf[100]; Biobuf *bin, *bout; - Type *t; + Type *t, *tt; Field *f; int orig_output_fd; @@ -373,8 +373,16 @@ Continue: prefix = prefixlen(t); for(j=0; j<t->nf; j++) { f = &t->f[j]; - if(f->type->kind == 0) - continue; + if(f->type->kind == 0 && f->size <= 64 && (f->size&(f->size-1)) == 0) { + // unknown type but <= 64 bits and bit size is a power of two. + // could be enum - make Uint64 and then let it reduce + tt = emalloc(sizeof *tt); + *tt = *f->type; + f->type = tt; + tt->kind = Uint64; + while(tt->kind > Uint8 && kindsize[tt->kind] > f->size) + tt->kind -= 2; + } // padding if(t->kind == Struct || lang == &go) { if(f->offset%8 != 0 || f->size%8 != 0) { diff --git a/src/cmd/godefs/stabs.c b/src/cmd/godefs/stabs.c index 30a05fc70..2c3d431b8 100644 --- a/src/cmd/godefs/stabs.c +++ b/src/cmd/godefs/stabs.c @@ -149,7 +149,7 @@ Intrange intranges[] = { 16, 0, Void, }; -static int kindsize[] = { +int kindsize[] = { 0, 0, 8, @@ -381,14 +381,6 @@ parsedef(char **pp, char *name) while(f->type->kind == Typedef) f->type = f->type->type; - if(f->type->kind == 0 && f->size <= 64 && (f->size&(f->size-1)) == 0) { - // unknown type but <= 64 bits and bit size is a power of two. - // could be enum - make Uint64 and then let it reduce - tt = emalloc(sizeof *tt); - *tt = *f->type; - f->type = tt; - tt->kind = Uint64; - } // rewrite // uint32 x : 8; diff --git a/src/cmd/godoc/doc.go b/src/cmd/godoc/doc.go index f0006e750..26d436d72 100644 --- a/src/cmd/godoc/doc.go +++ b/src/cmd/godoc/doc.go @@ -47,6 +47,9 @@ The flags are: width of tabs in units of spaces -timestamps=true show timestamps with directory listings + -index + enable identifier and full text search index + (no search box is shown if -index is not set) -maxresults=10000 maximum number of full text search results shown (no full text index is built if maxresults <= 0) diff --git a/src/cmd/godoc/godoc.go b/src/cmd/godoc/godoc.go index b8e9dbc92..f97c764f9 100644 --- a/src/cmd/godoc/godoc.go +++ b/src/cmd/godoc/godoc.go @@ -64,9 +64,12 @@ var ( // layout control tabwidth = flag.Int("tabwidth", 4, "tab width") showTimestamps = flag.Bool("timestamps", true, "show timestamps with directory listings") - maxResults = flag.Int("maxresults", 10000, "maximum number of full text search results shown") templateDir = flag.String("templates", "", "directory containing alternate template files") + // search index + indexEnabled = flag.Bool("index", false, "enable search index") + maxResults = flag.Int("maxresults", 10000, "maximum number of full text search results shown") + // file system mapping fsMap Mapping // user-defined mapping fsTree RWValue // *Directory tree of packages, updated with each sync @@ -687,17 +690,19 @@ func readTemplates() { func servePage(w http.ResponseWriter, title, subtitle, query string, content []byte) { d := struct { - Title string - Subtitle string - PkgRoots []string - Query string - Version string - Menu []byte - Content []byte + Title string + Subtitle string + PkgRoots []string + SearchBox bool + Query string + Version string + Menu []byte + Content []byte }{ title, subtitle, fsMap.PrefixList(), + *indexEnabled, query, runtime.Version(), nil, @@ -1174,11 +1179,15 @@ func lookup(query string) (result SearchResult) { } // is the result accurate? - if _, ts := fsModified.get(); timestamp < ts { - // The index is older than the latest file system change - // under godoc's observation. Indexing may be in progress - // or start shortly (see indexer()). - result.Alert = "Indexing in progress: result may be inaccurate" + if *indexEnabled { + if _, ts := fsModified.get(); timestamp < ts { + // The index is older than the latest file system change + // under godoc's observation. Indexing may be in progress + // or start shortly (see indexer()). + result.Alert = "Indexing in progress: result may be inaccurate" + } + } else { + result.Alert = "Search index disabled: no results available" } return diff --git a/src/cmd/godoc/main.go b/src/cmd/godoc/main.go index e426626b3..967ea8727 100644 --- a/src/cmd/godoc/main.go +++ b/src/cmd/godoc/main.go @@ -176,7 +176,7 @@ func remoteSearch(query string) (res *http.Response, err os.Error) { // remote search for _, addr := range addrs { url := "http://" + addr + search - res, _, err = http.Get(url) + res, err = http.Get(url) if err == nil && res.StatusCode == http.StatusOK { break } @@ -246,8 +246,13 @@ func main() { log.Printf("address = %s", *httpAddr) log.Printf("goroot = %s", *goroot) log.Printf("tabwidth = %d", *tabwidth) - if *maxResults > 0 { - log.Printf("maxresults = %d (full text index enabled)", *maxResults) + switch { + case !*indexEnabled: + log.Print("search index disabled") + case *maxResults > 0: + log.Printf("full text index enabled (maxresults = %d)", *maxResults) + default: + log.Print("identifier search index enabled") } if !fsMap.IsEmpty() { log.Print("user-defined mapping:") @@ -284,7 +289,9 @@ func main() { } // Start indexing goroutine. - go indexer() + if *indexEnabled { + go indexer() + } // Start http server. if err := http.ListenAndServe(*httpAddr, handler); err != nil { diff --git a/src/cmd/godoc/spec.go b/src/cmd/godoc/spec.go index f8b95e387..444e36e08 100644 --- a/src/cmd/godoc/spec.go +++ b/src/cmd/godoc/spec.go @@ -99,7 +99,8 @@ func (p *ebnfParser) parseTerm() bool { case token.STRING: p.next() - if p.tok == token.ELLIPSIS { + const ellipsis = "…" // U+2026, the horizontal ellipsis character + if p.tok == token.ILLEGAL && p.lit == ellipsis { p.next() p.expect(token.STRING) } @@ -128,6 +129,9 @@ func (p *ebnfParser) parseTerm() bool { func (p *ebnfParser) parseSequence() { + if !p.parseTerm() { + p.errorExpected(p.pos, "term") + } for p.parseTerm() { } } @@ -147,7 +151,9 @@ func (p *ebnfParser) parseExpression() { func (p *ebnfParser) parseProduction() { p.parseIdentifier(true) p.expect(token.ASSIGN) - p.parseExpression() + if p.tok != token.PERIOD { + p.parseExpression() + } p.expect(token.PERIOD) } @@ -157,7 +163,7 @@ func (p *ebnfParser) parse(fset *token.FileSet, out io.Writer, src []byte) { p.out = out p.src = src p.file = fset.AddFile("", fset.Base(), len(src)) - p.scanner.Init(p.file, src, p, 0) + p.scanner.Init(p.file, src, p, scanner.AllowIllegalChars) p.next() // initializes pos, tok, lit // process source diff --git a/src/cmd/gofix/Makefile b/src/cmd/gofix/Makefile index 12f09b4e4..c4127d93e 100644 --- a/src/cmd/gofix/Makefile +++ b/src/cmd/gofix/Makefile @@ -10,9 +10,11 @@ GOFILES=\ netdial.go\ main.go\ osopen.go\ + httpfinalurl.go\ httpserver.go\ procattr.go\ reflect.go\ + signal.go\ typecheck.go\ include ../../Make.cmd diff --git a/src/cmd/gofix/fix.go b/src/cmd/gofix/fix.go index 0852ce21e..c1c5a746c 100644 --- a/src/cmd/gofix/fix.go +++ b/src/cmd/gofix/fix.go @@ -10,6 +10,7 @@ import ( "go/token" "os" "strconv" + "strings" ) type fix struct { @@ -258,13 +259,28 @@ func walkBeforeAfter(x interface{}, before, after func(interface{})) { // imports returns true if f imports path. func imports(f *ast.File, path string) bool { + return importSpec(f, path) != nil +} + +// importSpec returns the import spec if f imports path, +// or nil otherwise. +func importSpec(f *ast.File, path string) *ast.ImportSpec { for _, s := range f.Imports { - t, err := strconv.Unquote(s.Path.Value) - if err == nil && t == path { - return true + if importPath(s) == path { + return s } } - return false + return nil +} + +// importPath returns the unquoted import path of s, +// or "" if the path is not properly quoted. +func importPath(s *ast.ImportSpec) string { + t, err := strconv.Unquote(s.Path.Value) + if err == nil { + return t + } + return "" } // isPkgDot returns true if t is the expression "pkg.name" @@ -420,3 +436,138 @@ func newPkgDot(pos token.Pos, pkg, name string) ast.Expr { }, } } + +// addImport adds the import path to the file f, if absent. +func addImport(f *ast.File, path string) { + if imports(f, path) { + return + } + + newImport := &ast.ImportSpec{ + Path: &ast.BasicLit{ + Kind: token.STRING, + Value: strconv.Quote(path), + }, + } + + var impdecl *ast.GenDecl + + // Find an import decl to add to. + for _, decl := range f.Decls { + gen, ok := decl.(*ast.GenDecl) + + if ok && gen.Tok == token.IMPORT { + impdecl = gen + break + } + } + + // No import decl found. Add one. + if impdecl == nil { + impdecl = &ast.GenDecl{ + Tok: token.IMPORT, + } + f.Decls = append(f.Decls, nil) + copy(f.Decls[1:], f.Decls) + f.Decls[0] = impdecl + } + + // Ensure the import decl has parentheses, if needed. + if len(impdecl.Specs) > 0 && !impdecl.Lparen.IsValid() { + impdecl.Lparen = impdecl.Pos() + } + + // Assume the import paths are alphabetically ordered. + // If they are not, the result is ugly, but legal. + insertAt := len(impdecl.Specs) // default to end of specs + for i, spec := range impdecl.Specs { + impspec := spec.(*ast.ImportSpec) + if importPath(impspec) > path { + insertAt = i + break + } + } + + impdecl.Specs = append(impdecl.Specs, nil) + copy(impdecl.Specs[insertAt+1:], impdecl.Specs[insertAt:]) + impdecl.Specs[insertAt] = newImport + + f.Imports = append(f.Imports, newImport) +} + +// deleteImport deletes the import path from the file f, if present. +func deleteImport(f *ast.File, path string) { + oldImport := importSpec(f, path) + + // Find the import node that imports path, if any. + for i, decl := range f.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.IMPORT { + continue + } + for j, spec := range gen.Specs { + impspec := spec.(*ast.ImportSpec) + + if oldImport != impspec { + continue + } + + // We found an import spec that imports path. + // Delete it. + copy(gen.Specs[j:], gen.Specs[j+1:]) + gen.Specs = gen.Specs[:len(gen.Specs)-1] + + // If this was the last import spec in this decl, + // delete the decl, too. + if len(gen.Specs) == 0 { + copy(f.Decls[i:], f.Decls[i+1:]) + f.Decls = f.Decls[:len(f.Decls)-1] + } else if len(gen.Specs) == 1 { + gen.Lparen = token.NoPos // drop parens + } + + break + } + } + + // Delete it from f.Imports. + for i, imp := range f.Imports { + if imp == oldImport { + copy(f.Imports[i:], f.Imports[i+1:]) + f.Imports = f.Imports[:len(f.Imports)-1] + break + } + } +} + +func usesImport(f *ast.File, path string) (used bool) { + spec := importSpec(f, path) + if spec == nil { + return + } + + name := spec.Name.String() + switch name { + case "<nil>": + // If the package name is not explicitly specified, + // make an educated guess. This is not guaranteed to be correct. + lastSlash := strings.LastIndex(path, "/") + if lastSlash == -1 { + name = path + } else { + name = path[lastSlash+1:] + } + case "_", ".": + // Not sure if this import is used - err on the side of caution. + return true + } + + walk(f, func(n interface{}) { + sel, ok := n.(*ast.SelectorExpr) + if ok && isTopName(sel.X, name) { + used = true + } + }) + + return +} diff --git a/src/cmd/gofix/httpfinalurl.go b/src/cmd/gofix/httpfinalurl.go new file mode 100644 index 000000000..53642b22f --- /dev/null +++ b/src/cmd/gofix/httpfinalurl.go @@ -0,0 +1,56 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +var httpFinalURLFix = fix{ + "httpfinalurl", + httpfinalurl, + `Adapt http Get calls to not have a finalURL result parameter. + + http://codereview.appspot.com/4535056/ +`, +} + +func init() { + register(httpFinalURLFix) +} + +func httpfinalurl(f *ast.File) bool { + if !imports(f, "http") { + return false + } + + fixed := false + walk(f, func(n interface{}) { + // Fix up calls to http.Get. + // + // If they have blank identifiers, remove them: + // resp, _, err := http.Get(url) + // -> resp, err := http.Get(url) + // + // But if they're using the finalURL parameter, warn: + // resp, finalURL, err := http.Get(url) + as, ok := n.(*ast.AssignStmt) + if !ok || len(as.Lhs) != 3 || len(as.Rhs) != 1 { + return + } + + if !isCall(as.Rhs[0], "http", "Get") { + return + } + + if isBlank(as.Lhs[1]) { + as.Lhs = []ast.Expr{as.Lhs[0], as.Lhs[2]} + fixed = true + } else { + warn(as.Pos(), "call to http.Get records final URL") + } + }) + return fixed +} diff --git a/src/cmd/gofix/httpfinalurl_test.go b/src/cmd/gofix/httpfinalurl_test.go new file mode 100644 index 000000000..9e7d6242d --- /dev/null +++ b/src/cmd/gofix/httpfinalurl_test.go @@ -0,0 +1,37 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(httpfinalurlTests) +} + +var httpfinalurlTests = []testCase{ + { + Name: "finalurl.0", + In: `package main + +import ( + "http" +) + +func f() { + resp, _, err := http.Get("http://www.google.com/") + _, _ = resp, err +} +`, + Out: `package main + +import ( + "http" +) + +func f() { + resp, err := http.Get("http://www.google.com/") + _, _ = resp, err +} +`, + }, +} diff --git a/src/cmd/gofix/main.go b/src/cmd/gofix/main.go index 4f7e923e3..1b091c18a 100644 --- a/src/cmd/gofix/main.go +++ b/src/cmd/gofix/main.go @@ -248,17 +248,11 @@ func diff(b1, b2 []byte) (data []byte, err os.Error) { f1.Write(b1) f2.Write(b2) - diffcmd, err := exec.LookPath("diff") - if err != nil { - return nil, err + data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput() + if len(data) > 0 { + // diff exits with a non-zero status when the files don't match. + // Ignore that failure as long as we get output. + err = nil } - - c, err := exec.Run(diffcmd, []string{"diff", f1.Name(), f2.Name()}, nil, "", - exec.DevNull, exec.Pipe, exec.MergeWithStdout) - if err != nil { - return nil, err - } - defer c.Close() - - return ioutil.ReadAll(c.Stdout) + return } diff --git a/src/cmd/gofix/signal.go b/src/cmd/gofix/signal.go new file mode 100644 index 000000000..53c338851 --- /dev/null +++ b/src/cmd/gofix/signal.go @@ -0,0 +1,49 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" + "strings" +) + +func init() { + register(fix{ + "signal", + signal, + `Adapt code to types moved from os/signal to signal. + +http://codereview.appspot.com/4437091 +`, + }) +} + +func signal(f *ast.File) (fixed bool) { + if !imports(f, "os/signal") { + return + } + + walk(f, func(n interface{}) { + s, ok := n.(*ast.SelectorExpr) + + if !ok || !isTopName(s.X, "signal") { + return + } + + sel := s.Sel.String() + if sel == "Signal" || sel == "UnixSignal" || strings.HasPrefix(sel, "SIG") { + s.X = &ast.Ident{Name: "os"} + fixed = true + } + }) + + if fixed { + addImport(f, "os") + if !usesImport(f, "os/signal") { + deleteImport(f, "os/signal") + } + } + return +} diff --git a/src/cmd/gofix/signal_test.go b/src/cmd/gofix/signal_test.go new file mode 100644 index 000000000..2500e9cee --- /dev/null +++ b/src/cmd/gofix/signal_test.go @@ -0,0 +1,96 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(signalTests) +} + +var signalTests = []testCase{ + { + Name: "signal.0", + In: `package main + +import ( + _ "a" + "os/signal" + _ "z" +) + +type T1 signal.UnixSignal +type T2 signal.Signal + +func f() { + _ = signal.SIGHUP + _ = signal.Incoming +} +`, + Out: `package main + +import ( + _ "a" + "os" + "os/signal" + _ "z" +) + +type T1 os.UnixSignal +type T2 os.Signal + +func f() { + _ = os.SIGHUP + _ = signal.Incoming +} +`, + }, + { + Name: "signal.1", + In: `package main + +import ( + "os" + "os/signal" +) + +func f() { + var _ os.Error + _ = signal.SIGHUP +} +`, + Out: `package main + +import "os" + + +func f() { + var _ os.Error + _ = os.SIGHUP +} +`, + }, + { + Name: "signal.2", + In: `package main + +import "os" +import "os/signal" + +func f() { + var _ os.Error + _ = signal.SIGHUP +} +`, + Out: `package main + +import "os" + + +func f() { + var _ os.Error + _ = os.SIGHUP +} +`, + }, +} diff --git a/src/cmd/gofmt/gofmt.go b/src/cmd/gofmt/gofmt.go index 5dd801d90..ea1c1b00f 100644 --- a/src/cmd/gofmt/gofmt.go +++ b/src/cmd/gofmt/gofmt.go @@ -245,14 +245,14 @@ func gofmtMain() { func diff(b1, b2 []byte) (data []byte, err os.Error) { f1, err := ioutil.TempFile("", "gofmt") if err != nil { - return nil, err + return } defer os.Remove(f1.Name()) defer f1.Close() f2, err := ioutil.TempFile("", "gofmt") if err != nil { - return nil, err + return } defer os.Remove(f2.Name()) defer f2.Close() @@ -260,17 +260,12 @@ func diff(b1, b2 []byte) (data []byte, err os.Error) { f1.Write(b1) f2.Write(b2) - diffcmd, err := exec.LookPath("diff") - if err != nil { - return nil, err - } - - c, err := exec.Run(diffcmd, []string{"diff", "-u", f1.Name(), f2.Name()}, - nil, "", exec.DevNull, exec.Pipe, exec.MergeWithStdout) - if err != nil { - return nil, err + data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput() + if len(data) > 0 { + // diff exits with a non-zero status when the files don't match. + // Ignore that failure as long as we get output. + err = nil } - defer c.Close() + return - return ioutil.ReadAll(c.Stdout) } diff --git a/src/cmd/gofmt/test.sh b/src/cmd/gofmt/test.sh index 99ec76932..5dce2ed7a 100755 --- a/src/cmd/gofmt/test.sh +++ b/src/cmd/gofmt/test.sh @@ -36,13 +36,14 @@ apply1() { # the following files are skipped because they are test cases # for syntax errors and thus won't parse in the first place: case `basename "$F"` in - func3.go | const2.go | char_lit1.go | blank1.go | \ + func3.go | const2.go | char_lit1.go | blank1.go | ddd1.go | \ bug014.go | bug050.go | bug068.go | bug083.go | bug088.go | \ bug106.go | bug121.go | bug125.go | bug133.go | bug160.go | \ bug163.go | bug166.go | bug169.go | bug217.go | bug222.go | \ bug226.go | bug228.go | bug248.go | bug274.go | bug280.go | \ bug282.go | bug287.go | bug298.go | bug299.go | bug300.go | \ - bug302.go | bug306.go | bug322.go | bug324.go ) return ;; + bug302.go | bug306.go | bug322.go | bug324.go | bug335.go | \ + bug340.go ) return ;; esac # the following directories are skipped because they contain test # cases for syntax errors and thus won't parse in the first place: diff --git a/src/cmd/goinstall/doc.go b/src/cmd/goinstall/doc.go index 15845b574..13c37d0a2 100644 --- a/src/cmd/goinstall/doc.go +++ b/src/cmd/goinstall/doc.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. /* - Goinstall is an experiment in automatic package installation. It installs packages, possibly downloading them from the internet. It maintains a list of public Go packages at http://godashboard.appspot.com/package. @@ -100,5 +99,59 @@ Instead, it invokes "make install" after locating the package sources. For local packages without a Makefile and all remote packages, goinstall creates and uses a temporary Makefile constructed from the import path and the list of Go files in the package. + + +The GOPATH Environment Variable + +GOPATH may be set to a colon-separated list of paths inside which Go code, +package objects, and executables may be found. + +Set a GOPATH to use goinstall to build and install your own code and +external libraries outside of the Go tree (and to avoid writing Makefiles). + +The top-level directory structure of a GOPATH is prescribed: + +The 'src' directory is for source code. The directory naming inside 'src' +determines the package import path or executable name. + +The 'pkg' directory is for package objects. Like the Go tree, package objects +are stored inside a directory named after the target operating system and +processor architecture ('pkg/$GOOS_$GOARCH'). +A package whose source is located at '$GOPATH/src/foo/bar' would be imported +as 'foo/bar' and installed as '$GOPATH/pkg/$GOOS_$GOARCH/foo/bar.a'. + +The 'bin' directory is for executable files. +Goinstall installs program binaries using the name of the source folder. +A binary whose source is at 'src/foo/qux' would be built and installed to +'$GOPATH/bin/qux'. (Note 'bin/qux', not 'bin/foo/qux' - this is such that +you can put the bin directory in your PATH.) + +Here's an example directory layout: + + GOPATH=/home/user/gocode + + /home/user/gocode/ + src/foo/ + bar/ (go code in package bar) + qux/ (go code in package main) + bin/qux (executable file) + pkg/linux_amd64/foo/bar.a (object file) + +Run 'goinstall foo/bar' to build and install the package 'foo/bar' +(and its dependencies). +Goinstall will search each GOPATH (in order) for 'src/foo/bar'. +If the directory cannot be found, goinstall will attempt to fetch the +source from a remote repository and write it to the 'src' directory of the +first GOPATH (or $GOROOT/src/pkg if GOPATH is not set). + +Goinstall recognizes relative and absolute paths (paths beginning with / or .). +The following commands would build our example packages: + + goinstall /home/user/gocode/src/foo/bar # build and install foo/bar + cd /home/user/gocode/src/foo + goinstall ./bar # build and install foo/bar (again) + cd qux + goinstall . # build and install foo/qux + */ package documentation diff --git a/src/cmd/goinstall/download.go b/src/cmd/goinstall/download.go index 7dad596ab..2edf85efd 100644 --- a/src/cmd/goinstall/download.go +++ b/src/cmd/goinstall/download.go @@ -31,23 +31,36 @@ func maybeReportToDashboard(path string) { } } -var googlecode = regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/(svn|hg))(/[a-z0-9A-Z_.\-/]*)?$`) -var github = regexp.MustCompile(`^(github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`) -var bitbucket = regexp.MustCompile(`^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`) -var launchpad = regexp.MustCompile(`^(launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+))(/[a-z0-9A-Z_.\-/]+)?$`) +var vcsPatterns = map[string]*regexp.Regexp{ + "googlecode": regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/(svn|hg))(/[a-z0-9A-Z_.\-/]*)?$`), + "github": regexp.MustCompile(`^(github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`), + "bitbucket": regexp.MustCompile(`^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`), + "launchpad": regexp.MustCompile(`^(launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+))(/[a-z0-9A-Z_.\-/]+)?$`), +} + +// isRemote returns true if the provided package path +// matches one of the supported remote repositories. +func isRemote(pkg string) bool { + for _, r := range vcsPatterns { + if r.MatchString(pkg) { + return true + } + } + return false +} // download checks out or updates pkg from the remote server. func download(pkg, srcDir string) os.Error { if strings.Contains(pkg, "..") { return os.ErrorString("invalid path (contains ..)") } - if m := bitbucket.FindStringSubmatch(pkg); m != nil { + if m := vcsPatterns["bitbucket"].FindStringSubmatch(pkg); m != nil { if err := vcsCheckout(&hg, srcDir, m[1], "http://"+m[1], m[1]); err != nil { return err } return nil } - if m := googlecode.FindStringSubmatch(pkg); m != nil { + if m := vcsPatterns["googlecode"].FindStringSubmatch(pkg); m != nil { var v *vcs switch m[2] { case "hg": @@ -63,7 +76,7 @@ func download(pkg, srcDir string) os.Error { } return nil } - if m := github.FindStringSubmatch(pkg); m != nil { + if m := vcsPatterns["github"].FindStringSubmatch(pkg); m != nil { if strings.HasSuffix(m[1], ".git") { return os.ErrorString("repository " + pkg + " should not have .git suffix") } @@ -72,7 +85,7 @@ func download(pkg, srcDir string) os.Error { } return nil } - if m := launchpad.FindStringSubmatch(pkg); m != nil { + if m := vcsPatterns["launchpad"].FindStringSubmatch(pkg); m != nil { // Either lp.net/<project>[/<series>[/<path>]] // or lp.net/~<user or team>/<project>/<branch>[/<path>] if err := vcsCheckout(&bzr, srcDir, m[1], "https://"+m[1], m[1]); err != nil { diff --git a/src/cmd/goinstall/main.go b/src/cmd/goinstall/main.go index 6cd92907a..721e719d2 100644 --- a/src/cmd/goinstall/main.go +++ b/src/cmd/goinstall/main.go @@ -12,7 +12,6 @@ import ( "flag" "fmt" "go/token" - "io" "io/ioutil" "os" "path/filepath" @@ -32,9 +31,8 @@ var ( argv0 = os.Args[0] errors = false parents = make(map[string]string) - root = runtime.GOROOT() visit = make(map[string]status) - logfile = filepath.Join(root, "goinstall.log") + logfile = filepath.Join(runtime.GOROOT(), "goinstall.log") installedPkgs = make(map[string]bool) allpkg = flag.Bool("a", false, "install all previously installed packages") @@ -52,14 +50,30 @@ const ( done ) +func logf(format string, args ...interface{}) { + format = "%s: " + format + args = append([]interface{}{argv0}, args...) + fmt.Fprintf(os.Stderr, format, args...) +} + +func vlogf(format string, args ...interface{}) { + if *verbose { + logf(format, args...) + } +} + +func errorf(format string, args ...interface{}) { + errors = true + logf(format, args...) +} + func main() { flag.Usage = usage flag.Parse() - if root == "" { + if runtime.GOROOT() == "" { fmt.Fprintf(os.Stderr, "%s: no $GOROOT\n", argv0) os.Exit(1) } - root += filepath.FromSlash("/src/pkg/") // special case - "unsafe" is already installed visit["unsafe"] = done @@ -88,6 +102,11 @@ func main() { usage() } for _, path := range args { + if strings.HasPrefix(path, "http://") { + errorf("'http://' used in remote path, try '%s'\n", path[7:]) + continue + } + install(path, "") } if errors { @@ -143,49 +162,44 @@ func install(pkg, parent string) { } visit[pkg] = visiting parents[pkg] = parent - if *verbose { - fmt.Println(pkg) - } + + vlogf("%s: visit\n", pkg) // Check whether package is local or remote. // If remote, download or update it. - var dir string - proot := gopath[0] // default to GOROOT - local := false - if strings.HasPrefix(pkg, "http://") { - fmt.Fprintf(os.Stderr, "%s: %s: 'http://' used in remote path, try '%s'\n", argv0, pkg, pkg[7:]) - errors = true + proot, pkg, err := findPackageRoot(pkg) + // Don't build the standard library. + if err == nil && proot.goroot && isStandardPath(pkg) { + if parent == "" { + errorf("%s: can not goinstall the standard library\n", pkg) + } else { + vlogf("%s: skipping standard library\n", pkg) + } + visit[pkg] = done return } - if isLocalPath(pkg) { - dir = pkg - local = true - } else if isStandardPath(pkg) { - dir = filepath.Join(root, filepath.FromSlash(pkg)) - local = true - } else { - proot = findPkgroot(pkg) - err := download(pkg, proot.srcDir()) - dir = filepath.Join(proot.srcDir(), pkg) - if err != nil { - fmt.Fprintf(os.Stderr, "%s: %s: %s\n", argv0, pkg, err) - errors = true - visit[pkg] = done - return - } + // Download remote packages if not found or forced with -u flag. + remote := isRemote(pkg) + if remote && (err == ErrPackageNotFound || (err == nil && *update)) { + vlogf("%s: download\n", pkg) + err = download(pkg, proot.srcDir()) } + if err != nil { + errorf("%s: %v\n", pkg, err) + visit[pkg] = done + return + } + dir := filepath.Join(proot.srcDir(), pkg) // Install prerequisites. dirInfo, err := scanDir(dir, parent == "") if err != nil { - fmt.Fprintf(os.Stderr, "%s: %s: %s\n", argv0, pkg, err) - errors = true + errorf("%s: %v\n", pkg, err) visit[pkg] = done return } if len(dirInfo.goFiles) == 0 { - fmt.Fprintf(os.Stderr, "%s: %s: package has no files\n", argv0, pkg) - errors = true + errorf("%s: package has no files\n", pkg) visit[pkg] = done return } @@ -198,22 +212,16 @@ func install(pkg, parent string) { // Install this package. if !errors { isCmd := dirInfo.pkgName == "main" - if err := domake(dir, pkg, proot, local, isCmd); err != nil { - fmt.Fprintf(os.Stderr, "%s: installing %s: %s\n", argv0, pkg, err) - errors = true - } else if !local && *logPkgs { - // mark this package as installed in $GOROOT/goinstall.log + if err := domake(dir, pkg, proot, isCmd); err != nil { + errorf("installing: %v\n", err) + } else if remote && *logPkgs { + // mark package as installed in $GOROOT/goinstall.log logPackage(pkg) } } visit[pkg] = done } -// Is this a local path? /foo ./foo ../foo . .. -func isLocalPath(s string) bool { - const sep = string(filepath.Separator) - return strings.HasPrefix(s, sep) || strings.HasPrefix(s, "."+sep) || strings.HasPrefix(s, ".."+sep) || s == "." || s == ".." -} // Is this a standard package path? strings container/vector etc. // Assume that if the first element has a dot, it's a domain name @@ -237,40 +245,22 @@ func quietRun(dir string, stdin []byte, cmd ...string) os.Error { } // genRun implements run and quietRun. -func genRun(dir string, stdin []byte, cmd []string, quiet bool) os.Error { - bin, err := exec.LookPath(cmd[0]) +func genRun(dir string, stdin []byte, arg []string, quiet bool) os.Error { + cmd := exec.Command(arg[0], arg[1:]...) + cmd.Stdin = bytes.NewBuffer(stdin) + cmd.Dir = dir + vlogf("%s: %s %s\n", dir, cmd.Path, strings.Join(arg[1:], " ")) + out, err := cmd.CombinedOutput() if err != nil { - return err - } - p, err := exec.Run(bin, cmd, os.Environ(), dir, exec.Pipe, exec.Pipe, exec.MergeWithStdout) - if *verbose { - fmt.Fprintf(os.Stderr, "%s: %s; %s %s\n", argv0, dir, bin, strings.Join(cmd[1:], " ")) - } - if err != nil { - return err - } - go func() { - p.Stdin.Write(stdin) - p.Stdin.Close() - }() - var buf bytes.Buffer - io.Copy(&buf, p.Stdout) - io.Copy(&buf, p.Stdout) - w, err := p.Wait(0) - p.Close() - if err != nil { - return err - } - if !w.Exited() || w.ExitStatus() != 0 { if !quiet || *verbose { if dir != "" { dir = "cd " + dir + "; " } - fmt.Fprintf(os.Stderr, "%s: === %s%s\n", argv0, dir, strings.Join(cmd, " ")) - os.Stderr.Write(buf.Bytes()) - fmt.Fprintf(os.Stderr, "--- %s\n", w) + fmt.Fprintf(os.Stderr, "%s: === %s%s\n", cmd.Path, dir, strings.Join(cmd.Args, " ")) + os.Stderr.Write(out) + fmt.Fprintf(os.Stderr, "--- %s\n", err) } - return os.ErrorString("running " + cmd[0] + ": " + w.String()) + return os.ErrorString("running " + arg[0] + ": " + err.String()) } return nil } diff --git a/src/cmd/goinstall/make.go b/src/cmd/goinstall/make.go index b2ca82b46..0c44481d7 100644 --- a/src/cmd/goinstall/make.go +++ b/src/cmd/goinstall/make.go @@ -14,26 +14,14 @@ import ( ) // domake builds the package in dir. -// If local is false, the package was copied from an external system. -// For non-local packages or packages without Makefiles, // domake generates a standard Makefile and passes it // to make on standard input. -func domake(dir, pkg string, root *pkgroot, local, isCmd bool) (err os.Error) { - needMakefile := true - if local { - _, err := os.Stat(dir + "/Makefile") - if err == nil { - needMakefile = false - } - } - cmd := []string{"gomake"} - var makefile []byte - if needMakefile { - if makefile, err = makeMakefile(dir, pkg, root, isCmd); err != nil { - return err - } - cmd = append(cmd, "-f-") +func domake(dir, pkg string, root *pkgroot, isCmd bool) (err os.Error) { + makefile, err := makeMakefile(dir, pkg, root, isCmd) + if err != nil { + return err } + cmd := []string{"bash", "gomake", "-f-"} if *clean { cmd = append(cmd, "clean") } @@ -51,16 +39,8 @@ func makeMakefile(dir, pkg string, root *pkgroot, isCmd bool) ([]byte, os.Error) targ := pkg targDir := root.pkgDir() if isCmd { - // use the last part of the package name only + // use the last part of the package name for targ _, targ = filepath.Split(pkg) - // if building the working dir use the directory name - if targ == "." { - d, err := filepath.Abs(dir) - if err != nil { - return nil, os.NewError("finding path: " + err.String()) - } - _, targ = filepath.Split(d) - } targDir = root.binDir() } dirInfo, err := scanDir(dir, isCmd) diff --git a/src/cmd/goinstall/path.go b/src/cmd/goinstall/path.go index 1153e0471..b8c392931 100644 --- a/src/cmd/goinstall/path.go +++ b/src/cmd/goinstall/path.go @@ -5,10 +5,12 @@ package main import ( + "fmt" "log" "os" "path/filepath" "runtime" + "strings" ) var ( @@ -19,6 +21,7 @@ var ( // set up gopath: parse and validate GOROOT and GOPATH variables func init() { + root := runtime.GOROOT() p, err := newPkgroot(root) if err != nil { log.Fatalf("Invalid GOROOT %q: %v", root, err) @@ -105,13 +108,42 @@ func (r *pkgroot) hasPkg(name string) bool { // TODO(adg): check object version is consistent } -// findPkgroot searches each of the gopath roots -// for the source code for the given import path. -func findPkgroot(importPath string) *pkgroot { + +var ErrPackageNotFound = os.NewError("package could not be found locally") + +// findPackageRoot takes an import or filesystem path and returns the +// root where the package source should be and the package import path. +func findPackageRoot(path string) (root *pkgroot, pkg string, err os.Error) { + if isLocalPath(path) { + if path, err = filepath.Abs(path); err != nil { + return + } + for _, r := range gopath { + rpath := r.srcDir() + string(filepath.Separator) + if !strings.HasPrefix(path, rpath) { + continue + } + root = r + pkg = path[len(rpath):] + return + } + err = fmt.Errorf("path %q not inside a GOPATH", path) + return + } + root = defaultRoot + pkg = path for _, r := range gopath { - if r.hasSrcDir(importPath) { - return r + if r.hasSrcDir(path) { + root = r + return } } - return defaultRoot + err = ErrPackageNotFound + return +} + +// Is this a local path? /foo ./foo ../foo . .. +func isLocalPath(s string) bool { + const sep = string(filepath.Separator) + return strings.HasPrefix(s, sep) || strings.HasPrefix(s, "."+sep) || strings.HasPrefix(s, ".."+sep) || s == "." || s == ".." } diff --git a/src/cmd/gopack/ar.c b/src/cmd/gopack/ar.c index 017978ced..0b5e608c7 100644 --- a/src/cmd/gopack/ar.c +++ b/src/cmd/gopack/ar.c @@ -1532,25 +1532,7 @@ arwrite(int fd, Armember *bp) int page(Arfile *ap) { - Armember *bp; - - bp = ap->head; - if (!ap->paged) { /* not yet paged - create file */ - ap->fname = mktemp(ap->fname); - ap->fd = create(ap->fname, ORDWR|ORCLOSE, 0600); - if (ap->fd < 0) { - fprint(2,"gopack: can't create temp file\n"); - return 0; - } - ap->paged = 1; - } - if (!arwrite(ap->fd, bp)) /* write member and free buffer block */ - return 0; - ap->head = bp->next; - if (ap->tail == bp) - ap->tail = bp->next; - free(bp->member); - free(bp); + sysfatal("page"); return 1; } diff --git a/src/cmd/gotest/gotest.go b/src/cmd/gotest/gotest.go index a7ba8dd11..4cb3da23c 100644 --- a/src/cmd/gotest/gotest.go +++ b/src/cmd/gotest/gotest.go @@ -52,7 +52,7 @@ var ( xFlag bool ) -// elapsed returns time elapsed since gotest started. +// elapsed returns the number of seconds since gotest started. func elapsed() float64 { return float64(time.Nanoseconds()-start) / 1e9 } @@ -182,7 +182,7 @@ func getTestFileNames() { } } -// parseFiles parses the files and remembers the packages we find. +// parseFiles parses the files and remembers the packages we find. func parseFiles() { fileSet := token.NewFileSet() for _, f := range files { @@ -285,7 +285,8 @@ func doRun(argv []string, returnStdout bool) string { } cmd += `"` + v + `"` } - argv = []string{"sh", "-c", cmd} + command = "bash" + argv = []string{"bash", "-c", cmd} } var err os.Error argv[0], err = exec.LookPath(argv[0]) diff --git a/src/cmd/gotype/gotype.go b/src/cmd/gotype/gotype.go index 568467322..b6a23ae5f 100644 --- a/src/cmd/gotype/gotype.go +++ b/src/cmd/gotype/gotype.go @@ -178,8 +178,10 @@ func processPackage(fset *token.FileSet, files map[string]*ast.File) { report(err) return } - // TODO(gri): typecheck package - _ = pkg + _, err = types.Check(fset, pkg) + if err != nil { + report(err) + } } diff --git a/src/cmd/hgpatch/main.go b/src/cmd/hgpatch/main.go index 2dcb5234c..8ee3422e2 100644 --- a/src/cmd/hgpatch/main.go +++ b/src/cmd/hgpatch/main.go @@ -10,7 +10,6 @@ import ( "exec" "flag" "fmt" - "io" "io/ioutil" "os" "patch" @@ -333,6 +332,7 @@ func run(argv []string, input []byte) (out string, err os.Error) { err = os.EINVAL goto Error } + prog, ok := lookPathCache[argv[0]] if !ok { prog, err = exec.LookPath(argv[0]) @@ -341,40 +341,15 @@ func run(argv []string, input []byte) (out string, err os.Error) { } lookPathCache[argv[0]] = prog } - // fmt.Fprintf(os.Stderr, "%v\n", argv); - var cmd *exec.Cmd - if len(input) == 0 { - cmd, err = exec.Run(prog, argv, os.Environ(), "", exec.DevNull, exec.Pipe, exec.MergeWithStdout) - if err != nil { - goto Error - } - } else { - cmd, err = exec.Run(prog, argv, os.Environ(), "", exec.Pipe, exec.Pipe, exec.MergeWithStdout) - if err != nil { - goto Error - } - go func() { - cmd.Stdin.Write(input) - cmd.Stdin.Close() - }() - } - defer cmd.Close() - var buf bytes.Buffer - _, err = io.Copy(&buf, cmd.Stdout) - out = buf.String() - if err != nil { - cmd.Wait(0) - goto Error - } - w, err := cmd.Wait(0) - if err != nil { - goto Error + + cmd := exec.Command(prog, argv[1:]...) + if len(input) > 0 { + cmd.Stdin = bytes.NewBuffer(input) } - if !w.Exited() || w.ExitStatus() != 0 { - err = w - goto Error + bs, err := cmd.CombinedOutput() + if err == nil { + return string(bs), nil } - return Error: err = &runError{dup(argv), err} diff --git a/src/cmd/ld/data.c b/src/cmd/ld/data.c index 0cb2b2138..3f3faade0 100644 --- a/src/cmd/ld/data.c +++ b/src/cmd/ld/data.c @@ -804,6 +804,10 @@ dodata(void) diag("%s: no size", s->name); t = 1; } + if(t >= PtrSize) + t = rnd(t, PtrSize); + else if(t > 2) + t = rnd(t, 4); if(t & 1) ; else if(t & 2) @@ -826,6 +830,10 @@ dodata(void) diag("unexpected symbol type %d", s->type); } t = s->size; + if(t >= PtrSize) + t = rnd(t, PtrSize); + else if(t > 2) + t = rnd(t, 4); if(t & 1) ; else if(t & 2) @@ -899,10 +907,8 @@ address(void) segdata.fileoff = va - segtext.vaddr + segtext.fileoff; if(HEADTYPE == Hwindows) segdata.fileoff = segtext.fileoff + rnd(segtext.len, PEFILEALIGN); - if(HEADTYPE == Hplan9x32) { - segdata.vaddr = va = rnd(va, 4096); + if(HEADTYPE == Hplan9x32) segdata.fileoff = segtext.fileoff + segtext.filelen; - } for(s=segdata.sect; s != nil; s=s->next) { s->vaddr = va; va += s->len; diff --git a/src/cmd/ld/dwarf.c b/src/cmd/ld/dwarf.c index 98b068008..50b42183e 100644 --- a/src/cmd/ld/dwarf.c +++ b/src/cmd/ld/dwarf.c @@ -529,8 +529,10 @@ find_or_diag(DWDie *die, char* name) { DWDie *r; r = find(die, name); - if (r == nil) + if (r == nil) { diag("dwarf find: %s has no %s", getattr(die, DW_AT_name)->data, name); + errorexit(); + } return r; } @@ -613,7 +615,7 @@ putattr(int form, int cls, vlong value, char *data) case DW_FORM_ref_addr: // reference to a DIE in the .info section if (data == nil) { - diag("null dwarf reference"); + diag("dwarf: null reference"); LPUT(0); // invalid dwarf, gdb will complain. } else { if (((DWDie*)data)->offs == 0) @@ -631,7 +633,7 @@ putattr(int form, int cls, vlong value, char *data) case DW_FORM_strp: // string case DW_FORM_indirect: // (see Section 7.5.3) default: - diag("Unsupported atribute form %d / class %d", form, cls); + diag("dwarf: unsupported attribute form %d / class %d", form, cls); errorexit(); } } @@ -823,7 +825,7 @@ decode_inuxi(uchar* p, int sz) inuxi = inuxi8; break; default: - diag("decode inuxi %d", sz); + diag("dwarf: decode inuxi %d", sz); errorexit(); } for (i = 0; i < sz; i++) @@ -1013,7 +1015,7 @@ defgotype(Sym *gotype) return find_or_diag(&dwtypes, "<unspecified>"); if (strncmp("type.", gotype->name, 5) != 0) { - diag("Type name doesn't start with \".type\": %s", gotype->name); + diag("dwarf: type name doesn't start with \".type\": %s", gotype->name); return find_or_diag(&dwtypes, "<unspecified>"); } name = gotype->name + 5; // could also decode from Type.string @@ -1164,7 +1166,7 @@ defgotype(Sym *gotype) break; default: - diag("definition of unknown kind %d: %s", kind, gotype->name); + diag("dwarf: definition of unknown kind %d: %s", kind, gotype->name); die = newdie(&dwtypes, DW_ABRV_TYPEDECL, name); newrefattr(die, DW_AT_type, find_or_diag(&dwtypes, "<unspecified>")); } @@ -1346,7 +1348,7 @@ synthesizemaptypes(DWDie *die) valtype = defptrto(valtype); newrefattr(fld, DW_AT_type, valtype); newmemberoffsetattr(fld, hashsize + datavo); - newattr(dwhe, DW_AT_byte_size, DW_CLS_CONSTANT, hashsize + datsize, NULL); + newattr(dwhe, DW_AT_byte_size, DW_CLS_CONSTANT, hashsize + datsize, nil); // Construct hash_subtable<hash_entry<K,V>> dwhs = newdie(&dwtypes, DW_ABRV_STRUCTTYPE, @@ -1357,7 +1359,7 @@ synthesizemaptypes(DWDie *die) substitutetype(dwhs, "end", defptrto(dwhe)); substitutetype(dwhs, "entry", dwhe); // todo: []hash_entry with dynamic size newattr(dwhs, DW_AT_byte_size, DW_CLS_CONSTANT, - getattr(hash_subtable, DW_AT_byte_size)->value, NULL); + getattr(hash_subtable, DW_AT_byte_size)->value, nil); // Construct hash<K,V> dwh = newdie(&dwtypes, DW_ABRV_STRUCTTYPE, @@ -1367,7 +1369,7 @@ synthesizemaptypes(DWDie *die) copychildren(dwh, hash); substitutetype(dwh, "st", defptrto(dwhs)); newattr(dwh, DW_AT_byte_size, DW_CLS_CONSTANT, - getattr(hash, DW_AT_byte_size)->value, NULL); + getattr(hash, DW_AT_byte_size)->value, nil); newrefattr(die, DW_AT_type, defptrto(dwh)); } @@ -1399,30 +1401,30 @@ synthesizechantypes(DWDie *die) // sudog<T> dws = newdie(&dwtypes, DW_ABRV_STRUCTTYPE, mkinternaltypename("sudog", - getattr(elemtype, DW_AT_name)->data, NULL)); + getattr(elemtype, DW_AT_name)->data, nil)); copychildren(dws, sudog); substitutetype(dws, "elem", elemtype); newattr(dws, DW_AT_byte_size, DW_CLS_CONSTANT, - sudogsize + (elemsize > 8 ? elemsize - 8 : 0), NULL); + sudogsize + (elemsize > 8 ? elemsize - 8 : 0), nil); // waitq<T> dww = newdie(&dwtypes, DW_ABRV_STRUCTTYPE, - mkinternaltypename("waitq", getattr(elemtype, DW_AT_name)->data, NULL)); + mkinternaltypename("waitq", getattr(elemtype, DW_AT_name)->data, nil)); copychildren(dww, waitq); substitutetype(dww, "first", defptrto(dws)); substitutetype(dww, "last", defptrto(dws)); newattr(dww, DW_AT_byte_size, DW_CLS_CONSTANT, - getattr(waitq, DW_AT_byte_size)->value, NULL); + getattr(waitq, DW_AT_byte_size)->value, nil); // hchan<T> dwh = newdie(&dwtypes, DW_ABRV_STRUCTTYPE, - mkinternaltypename("hchan", getattr(elemtype, DW_AT_name)->data, NULL)); + mkinternaltypename("hchan", getattr(elemtype, DW_AT_name)->data, nil)); copychildren(dwh, hchan); substitutetype(dwh, "recvq", dww); substitutetype(dwh, "sendq", dww); substitutetype(dwh, "free", defptrto(dws)); newattr(dwh, DW_AT_byte_size, DW_CLS_CONSTANT, - getattr(hchan, DW_AT_byte_size)->value, NULL); + getattr(hchan, DW_AT_byte_size)->value, nil); newrefattr(die, DW_AT_type, defptrto(dwh)); } @@ -1434,6 +1436,7 @@ defdwsymb(Sym* sym, char *s, int t, vlong v, vlong size, int ver, Sym *gotype) { DWDie *dv, *dt; + USED(size); if (strncmp(s, "go.string.", 10) == 0) return; @@ -1513,12 +1516,12 @@ decodez(char *s) ss = s + 1; // first is 0 while((o = ((uint8)ss[0] << 8) | (uint8)ss[1]) != 0) { if (o < 0 || o >= ftabsize) { - diag("corrupt z entry"); + diag("dwarf: corrupt z entry"); return 0; } f = ftab[o]; if (f == nil) { - diag("corrupt z entry"); + diag("dwarf: corrupt z entry"); return 0; } len += strlen(f) + 1; // for the '/' @@ -1590,7 +1593,7 @@ addhistfile(char *zentry) // if the histfile stack contains ..../runtime/runtime_defs.go // use that to set gdbscript static void -finddebugruntimepath() +finddebugruntimepath(void) { int i, l; char *c; @@ -1630,11 +1633,11 @@ checknesting(void) int i; if (includetop < 0) { - diag("corrupt z stack"); + diag("dwarf: corrupt z stack"); errorexit(); } if (includetop >= nelem(includestack)) { - diag("nesting too deep"); + diag("dwarf: nesting too deep"); for (i = 0; i < nelem(includestack); i++) diag("\t%s", histfile[includestack[i].file]); errorexit(); @@ -1660,7 +1663,7 @@ inithist(Auto *a) // We have a new history. They are guaranteed to come completely // at the beginning of the compilation unit. if (a->aoffset != 1) { - diag("stray 'z' with offset %d", a->aoffset); + diag("dwarf: stray 'z' with offset %d", a->aoffset); return 0; } @@ -1801,7 +1804,7 @@ mkvarname(char* name, int da) // flush previous compilation unit. static void -flushunit(DWDie *dwinfo, vlong pc, vlong unitstart) +flushunit(DWDie *dwinfo, vlong pc, vlong unitstart, int32 header_length) { vlong here; @@ -1817,7 +1820,9 @@ flushunit(DWDie *dwinfo, vlong pc, vlong unitstart) here = cpos(); seek(cout, unitstart, 0); - LPUT(here - unitstart - sizeof(int32)); + LPUT(here - unitstart - sizeof(int32)); // unit_length + WPUT(3); // dwarf version + LPUT(header_length); // header lenght starting here cflush(); seek(cout, here, 0); } @@ -1829,7 +1834,7 @@ writelines(void) Prog *q; Sym *s; Auto *a; - vlong unitstart, offs; + vlong unitstart, headerend, offs; vlong pc, epc, lc, llc, lline; int currfile; int i, lang, da, dt; @@ -1839,7 +1844,9 @@ writelines(void) char *n, *nn; unitstart = -1; - epc = pc = 0; + headerend = -1; + pc = 0; + epc = 0; lc = 1; llc = 1; currfile = -1; @@ -1855,7 +1862,7 @@ writelines(void) // we're entering a new compilation unit if (inithist(s->autom)) { - flushunit(dwinfo, epc, unitstart); + flushunit(dwinfo, epc, unitstart, headerend - unitstart - 10); unitstart = cpos(); if(debug['v'] > 1) { @@ -1876,10 +1883,10 @@ writelines(void) // Write .debug_line Line Number Program Header (sec 6.2.4) // Fields marked with (*) must be changed for 64-bit dwarf - LPUT(0); // unit_length (*), will be filled in later. + LPUT(0); // unit_length (*), will be filled in by flushunit. WPUT(3); // dwarf version (appendix F) - LPUT(11); // header_length (*), starting here. - + LPUT(0); // header_length (*), filled in by flushunit. + // cpos == unitstart + 4 + 2 + 4 cput(1); // minimum_instruction_length cput(1); // default_is_stmt cput(LINE_BASE); // line_base @@ -1890,18 +1897,17 @@ writelines(void) cput(1); // standard_opcode_lengths[3] cput(1); // standard_opcode_lengths[4] cput(0); // include_directories (empty) - cput(0); // file_names (empty) (emitted by DW_LNE's below) - // header_length ends here. for (i=1; i < histfilesize; i++) { - cput(0); // start extended opcode - uleb128put(1 + strlen(histfile[i]) + 4); - cput(DW_LNE_define_file); strnput(histfile[i], strlen(histfile[i]) + 4); // 4 zeros: the string termination + 3 fields. } - epc = pc = s->text->pc; + cput(0); // terminate file_names. + headerend = cpos(); + + pc = s->text->pc; + epc = pc; currfile = 1; lc = 1; llc = 1; @@ -1915,7 +1921,7 @@ writelines(void) continue; if (unitstart < 0) { - diag("reachable code before seeing any history: %P", s->text); + diag("dwarf: reachable code before seeing any history: %P", s->text); continue; } @@ -1932,7 +1938,7 @@ writelines(void) for(q = s->text; q != P; q = q->link) { lh = searchhist(q->line); if (lh == nil) { - diag("corrupt history or bad absolute line: %P", q); + diag("dwarf: corrupt history or bad absolute line: %P", q); continue; } @@ -1990,7 +1996,7 @@ writelines(void) newrefattr(dwvar, DW_AT_type, defgotype(a->gotype)); // push dwvar down dwfunc->child to preserve order - newattr(dwvar, DW_AT_internal_location, DW_CLS_CONSTANT, offs, NULL); + newattr(dwvar, DW_AT_internal_location, DW_CLS_CONSTANT, offs, nil); dwfunc->child = dwvar->link; // take dwvar out from the top of the list for (dws = &dwfunc->child; *dws != nil; dws = &(*dws)->link) if (offs > getattr(*dws, DW_AT_internal_location)->value) @@ -2004,7 +2010,7 @@ writelines(void) dwfunc->hash = nil; } - flushunit(dwinfo, epc, unitstart); + flushunit(dwinfo, epc, unitstart, headerend - unitstart - 10); linesize = cpos() - lineo; } @@ -2066,7 +2072,7 @@ writeframes(void) // 4 is to exclude the length field. pad = CIERESERVE + frameo + 4 - cpos(); if (pad < 0) { - diag("CIERESERVE too small by %lld bytes.", -pad); + diag("dwarf: CIERESERVE too small by %lld bytes.", -pad); errorexit(); } strnput("", pad); @@ -2296,6 +2302,9 @@ dwarfemitdebugsections(void) vlong infoe; DWDie* die; + if(debug['w']) // disable dwarf + return; + // For diagnostic messages. newattr(&dwtypes, DW_AT_name, DW_CLS_STRING, strlen("dwtypes"), "dwtypes"); @@ -2340,7 +2349,11 @@ dwarfemitdebugsections(void) infoo = cpos(); writeinfo(); - gdbscripto = arangeso = pubtypeso = pubnameso = infoe = cpos(); + infoe = cpos(); + pubnameso = infoe; + pubtypeso = infoe; + arangeso = infoe; + gdbscripto = infoe; if (fwdcount > 0) { if (debug['v']) @@ -2348,11 +2361,11 @@ dwarfemitdebugsections(void) seek(cout, infoo, 0); writeinfo(); if (fwdcount > 0) { - diag("unresolved references after first dwarf info pass"); + diag("dwarf: unresolved references after first dwarf info pass"); errorexit(); } if (infoe != cpos()) { - diag("inconsistent second dwarf info pass"); + diag("dwarf: inconsistent second dwarf info pass"); errorexit(); } } @@ -2401,6 +2414,9 @@ vlong elfstrdbg[NElfStrDbg]; void dwarfaddshstrings(Sym *shstrtab) { + if(debug['w']) // disable dwarf + return; + elfstrdbg[ElfStrDebugAbbrev] = addstring(shstrtab, ".debug_abbrev"); elfstrdbg[ElfStrDebugAranges] = addstring(shstrtab, ".debug_aranges"); elfstrdbg[ElfStrDebugFrame] = addstring(shstrtab, ".debug_frame"); @@ -2420,6 +2436,9 @@ dwarfaddelfheaders(void) { ElfShdr *sh; + if(debug['w']) // disable dwarf + return; + sh = newElfShdr(elfstrdbg[ElfStrDebugAbbrev]); sh->type = SHT_PROGBITS; sh->off = abbrevo; @@ -2488,6 +2507,9 @@ dwarfaddmachoheaders(void) vlong fakestart; int nsect; + if(debug['w']) // disable dwarf + return; + // Zero vsize segments won't be loaded in memory, even so they // have to be page aligned in the file. fakestart = abbrevo & ~0xfff; @@ -2562,7 +2584,9 @@ dwarfaddmachoheaders(void) void dwarfaddpeheaders(void) { - dwarfemitdebugsections(); + if(debug['w']) // disable dwarf + return; + newPEDWARFSection(".debug_abbrev", abbrevsize); newPEDWARFSection(".debug_line", linesize); newPEDWARFSection(".debug_frame", framesize); diff --git a/src/cmd/ld/elf.h b/src/cmd/ld/elf.h index 08583cc8f..d1370d28b 100644 --- a/src/cmd/ld/elf.h +++ b/src/cmd/ld/elf.h @@ -978,6 +978,10 @@ ElfShdr* elfshbits(Section*); void elfsetstring(char*, int); void elfaddverneed(Sym*); +EXTERN int elfstrsize; +EXTERN char* elfstrdat; +EXTERN int elftextsh; + /* * Total amount of space to reserve at the start of the file * for Header, PHeaders, SHeaders, and interp. diff --git a/src/cmd/ld/go.c b/src/cmd/ld/go.c index e52c5cb34..05d1cc136 100644 --- a/src/cmd/ld/go.c +++ b/src/cmd/ld/go.c @@ -415,8 +415,8 @@ loaddynimport(char *file, char *pkg, char *p, int n) char *pend, *next, *name, *def, *p0, *lib, *q; Sym *s; + USED(file); pend = p + n; - p0 = p; for(; p<pend; p=next) { next = strchr(p, '\n'); if(next == nil) @@ -454,6 +454,7 @@ loaddynimport(char *file, char *pkg, char *p, int n) if(strcmp(name, "_") == 0 && strcmp(def, "_") == 0) { // allow #pragma dynimport _ _ "foo.so" // to force a link of foo.so. + havedynamic = 1; adddynlib(lib); continue; } @@ -468,6 +469,7 @@ loaddynimport(char *file, char *pkg, char *p, int n) s->dynimpname = def; s->dynimpvers = q; s->type = SDYNIMPORT; + havedynamic = 1; } } return; @@ -483,8 +485,8 @@ loaddynexport(char *file, char *pkg, char *p, int n) char *pend, *next, *local, *elocal, *remote, *p0; Sym *s; + USED(file); pend = p + n; - p0 = p; for(; p<pend; p=next) { next = strchr(p, '\n'); if(next == nil) diff --git a/src/cmd/ld/ldpe.c b/src/cmd/ld/ldpe.c index d8b0a6fc2..d6aa267c4 100644 --- a/src/cmd/ld/ldpe.c +++ b/src/cmd/ld/ldpe.c @@ -147,7 +147,7 @@ ldpe(Biobuf *f, char *pkg, int64 len, char *pn) goto bad; obj->sect[i].size = obj->sect[i].sh.SizeOfRawData; obj->sect[i].name = (char*)obj->sect[i].sh.Name; - // TODO return error if found .cormeta .rsrc + // TODO return error if found .cormeta } // load string table Bseek(f, base+obj->fh.PointerToSymbolTable+18*obj->fh.NumberOfSymbols, 0); @@ -222,6 +222,8 @@ ldpe(Biobuf *f, char *pkg, int64 len, char *pn) etextp = s; } sect->sym = s; + if(strcmp(sect->name, ".rsrc") == 0) + setpersrc(sect->sym); } // load relocations @@ -259,6 +261,7 @@ ldpe(Biobuf *f, char *pkg, int64 len, char *pn) rp->type = D_PCREL; rp->add = 0; break; + case IMAGE_REL_I386_DIR32NB: case IMAGE_REL_I386_DIR32: rp->type = D_ADDR; // load addend from image diff --git a/src/cmd/ld/lib.c b/src/cmd/ld/lib.c index 15219ba11..04ee790a4 100644 --- a/src/cmd/ld/lib.c +++ b/src/cmd/ld/lib.c @@ -69,7 +69,7 @@ libinit(void) // add goroot to the end of the libdir list. libdir[nlibdir++] = smprint("%s/pkg/%s_%s", goroot, goos, goarch); - unlink(outfile); + remove(outfile); cout = create(outfile, 1, 0775); if(cout < 0) { diag("cannot create %s", outfile); @@ -235,30 +235,52 @@ addlibpath(char *srcref, char *objref, char *file, char *pkg) } void -loadlib(void) +loadinternal(char *name) { char pname[1024]; int i, found; found = 0; for(i=0; i<nlibdir; i++) { - snprint(pname, sizeof pname, "%s/runtime.a", libdir[i]); + snprint(pname, sizeof pname, "%s/%s.a", libdir[i], name); if(debug['v']) - Bprint(&bso, "searching for runtime.a in %s\n", pname); + Bprint(&bso, "searching for %s.a in %s\n", name, pname); if(access(pname, AEXIST) >= 0) { - addlibpath("internal", "internal", pname, "runtime"); + addlibpath("internal", "internal", pname, name); found = 1; break; } } if(!found) - Bprint(&bso, "warning: unable to find runtime.a\n"); + Bprint(&bso, "warning: unable to find %s.a\n", name); +} + +void +loadlib(void) +{ + int i; + + loadinternal("runtime"); + if(thechar == '5') + loadinternal("math"); for(i=0; i<libraryp; i++) { if(debug['v']) Bprint(&bso, "%5.2f autolib: %s (from %s)\n", cputime(), library[i].file, library[i].objref); objfile(library[i].file, library[i].pkg); } + + // We've loaded all the code now. + // If there are no dynamic libraries needed, gcc disables dynamic linking. + // Because of this, glibc's dynamic ELF loader occasionally (like in version 2.13) + // assumes that a dynamic binary always refers to at least one dynamic library. + // Rather than be a source of test cases for glibc, disable dynamic linking + // the same way that gcc would. + // + // Exception: on OS X, programs such as Shark only work with dynamic + // binaries, so leave it enabled on OS X (Mach-O) binaries. + if(!havedynamic && HEADTYPE != Hdarwin) + debug['d'] = 1; } /* @@ -386,9 +408,6 @@ ldobj(Biobuf *f, char *pkg, int64 len, char *pn, int whence) eof = Boffset(f) + len; pn = strdup(pn); - - USED(c4); - USED(magic); c1 = Bgetc(f); c2 = Bgetc(f); @@ -398,7 +417,7 @@ ldobj(Biobuf *f, char *pkg, int64 len, char *pn, int whence) Bungetc(f); Bungetc(f); Bungetc(f); - + magic = c1<<24 | c2<<16 | c3<<8 | c4; if(magic == 0x7f454c46) { // \x7F E L F ldelf(f, pkg, len, pn); @@ -486,7 +505,6 @@ _lookup(char *symb, int v, int creat) // not if(h < 0) h = ~h, because gcc 4.3 -O2 miscompiles it. h &= 0xffffff; h %= NHASH; - c = symb[0]; for(s = hash[h]; s != S; s = s->hash) if(memcmp(s->name, symb, l) == 0) return s; @@ -511,7 +529,7 @@ _lookup(char *symb, int v, int creat) s->size = 0; hash[h] = s; nsymbol++; - + s->allsym = allsym; allsym = s; return s; @@ -538,7 +556,6 @@ copyhistfrog(char *buf, int nbuf) p = buf; ep = buf + nbuf; - i = 0; for(i=0; i<histfrogp; i++) { p = seprint(p, ep, "%s", histfrog[i]->name+1); if(i+1<histfrogp && (p == buf || p[-1] != '/')) diff --git a/src/cmd/ld/lib.h b/src/cmd/ld/lib.h index 8b603a04a..dfd18fbff 100644 --- a/src/cmd/ld/lib.h +++ b/src/cmd/ld/lib.h @@ -122,6 +122,7 @@ EXTERN char* outfile; EXTERN int32 nsymbol; EXTERN char* thestring; EXTERN int ndynexp; +EXTERN int havedynamic; EXTERN Segment segtext; EXTERN Segment segdata; @@ -185,7 +186,7 @@ vlong addsize(Sym*, Sym*); vlong adduint8(Sym*, uint8); vlong adduint16(Sym*, uint16); void asmsym(void); -void asmelfsym64(void); +void asmelfsym(void); void asmplan9sym(void); void strnput(char*, int); void dodata(void); @@ -200,6 +201,7 @@ void addexport(void); void dostkcheck(void); void undef(void); void doweak(void); +void setpersrc(Sym*); int pathchar(void); void* mal(uint32); diff --git a/src/cmd/ld/macho.c b/src/cmd/ld/macho.c index 01349bb10..0b12ac17b 100644 --- a/src/cmd/ld/macho.c +++ b/src/cmd/ld/macho.c @@ -17,6 +17,14 @@ static MachoSeg seg[16]; static MachoDebug xdebug[16]; static int nload, mload, nseg, ndebug, nsect; +// Amount of space left for adding load commands +// that refer to dynamic libraries. Because these have +// to go in the Mach-O header, we can't just pick a +// "big enough" header size. The initial header is +// one page, the non-dynamic library stuff takes +// up about 1300 bytes; we overestimate that as 2k. +static int load_budget = INITIAL_MACHO_HEADR - 2*1024; + void machoinit(void) { @@ -267,6 +275,17 @@ domacho(void) void machoadddynlib(char *lib) { + // Will need to store the library name rounded up + // and 24 bytes of header metadata. If not enough + // space, grab another page of initial space at the + // beginning of the output file. + load_budget -= (strlen(lib)+7)/8*8 + 24; + if(load_budget < 0) { + HEADR += 4096; + INITTEXT += 4096; + load_budget += 4096; + } + if(ndylib%32 == 0) { dylib = realloc(dylib, (ndylib+32)*sizeof dylib[0]); if(dylib == nil) { @@ -463,8 +482,8 @@ asmbmacho(void) } a = machowrite(); - if(a > MACHORESERVE) - diag("MACHORESERVE too small: %d > %d", a, MACHORESERVE); + if(a > HEADR) + diag("HEADR too small: %d > %d", a, HEADR); } vlong diff --git a/src/cmd/ld/macho.h b/src/cmd/ld/macho.h index 4cc7edc80..f55104150 100644 --- a/src/cmd/ld/macho.h +++ b/src/cmd/ld/macho.h @@ -63,7 +63,7 @@ void machoinit(void); * for Header, PHeaders, and SHeaders. * May waste some. */ -#define MACHORESERVE 3*1024 +#define INITIAL_MACHO_HEADR 4*1024 enum { MACHO_CPU_AMD64 = (1<<24)|7, diff --git a/src/cmd/ld/pe.c b/src/cmd/ld/pe.c index d523ca9c5..91e15d343 100644 --- a/src/cmd/ld/pe.c +++ b/src/cmd/ld/pe.c @@ -34,6 +34,8 @@ static char dosstub[] = 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static Sym *rsrcsym; + static char symnames[256]; static int nextsymoff; @@ -459,6 +461,48 @@ addsymtable(void) } void +setpersrc(Sym *sym) +{ + if(rsrcsym != nil) + diag("too many .rsrc sections"); + + rsrcsym = sym; +} + +void +addpersrc(void) +{ + IMAGE_SECTION_HEADER *h; + uchar *p; + uint32 val; + Reloc *r; + + if(rsrcsym == nil) + return; + + h = addpesection(".rsrc", rsrcsym->size, rsrcsym->size, 0); + h->Characteristics = IMAGE_SCN_MEM_READ| + IMAGE_SCN_MEM_WRITE | IMAGE_SCN_CNT_INITIALIZED_DATA; + // relocation + for(r=rsrcsym->r; r<rsrcsym->r+rsrcsym->nr; r++) { + p = rsrcsym->p + r->off; + val = h->VirtualAddress + r->add; + // 32-bit little-endian + p[0] = val; + p[1] = val>>8; + p[2] = val>>16; + p[3] = val>>24; + } + ewrite(cout, rsrcsym->p, rsrcsym->size); + strnput("", h->SizeOfRawData - rsrcsym->size); + cflush(); + + // update data directory + dd[IMAGE_DIRECTORY_ENTRY_RESOURCE].VirtualAddress = h->VirtualAddress; + dd[IMAGE_DIRECTORY_ENTRY_RESOURCE].Size = h->VirtualSize; +} + +void asmbpe(void) { IMAGE_SECTION_HEADER *t, *d; @@ -484,15 +528,17 @@ asmbpe(void) d->Characteristics = IMAGE_SCN_CNT_INITIALIZED_DATA| IMAGE_SCN_MEM_READ|IMAGE_SCN_MEM_WRITE; + if(!debug['s']) + dwarfaddpeheaders(); + addimports(nextfileoff, d); addexports(nextfileoff); - if(!debug['s']) - dwarfaddpeheaders(); - addsymtable(); - + + addpersrc(); + fh.NumberOfSections = nsect; fh.TimeDateStamp = time(0); fh.Characteristics = IMAGE_FILE_RELOCS_STRIPPED| diff --git a/src/cmd/ld/pe.h b/src/cmd/ld/pe.h index 2180fb88c..7aa938829 100644 --- a/src/cmd/ld/pe.h +++ b/src/cmd/ld/pe.h @@ -175,3 +175,5 @@ typedef struct { uint32 NumberOfRvaAndSizes; IMAGE_DATA_DIRECTORY DataDirectory[16]; } PE64_IMAGE_OPTIONAL_HEADER; + +void setpersrc(Sym *sym); diff --git a/src/cmd/ld/symtab.c b/src/cmd/ld/symtab.c index da698fcc0..e3093b2aa 100644 --- a/src/cmd/ld/symtab.c +++ b/src/cmd/ld/symtab.c @@ -61,49 +61,35 @@ putelfstr(char *s) } void -putelfsym64(Sym *x, char *s, int t, vlong addr, vlong size, int ver, Sym *go) +putelfsyment(int off, vlong addr, vlong size, int info, int shndx) { - int bind, type, shndx, stroff; - - bind = STB_GLOBAL; - switch(t) { - default: - return; - case 'T': - type = STT_FUNC; - shndx = elftextsh + 0; - break; - case 'D': - type = STT_OBJECT; - shndx = elftextsh + 1; + switch(thechar) { + case '6': + LPUT(off); + cput(info); + cput(0); + WPUT(shndx); + VPUT(addr); + VPUT(size); + symsize += ELF64SYMSIZE; break; - case 'B': - type = STT_OBJECT; - shndx = elftextsh + 2; + default: + LPUT(off); + LPUT(addr); + LPUT(size); + cput(info); + cput(0); + WPUT(shndx); + symsize += ELF32SYMSIZE; break; } - - stroff = putelfstr(s); - LPUT(stroff); // string - cput((bind<<4)|(type&0xF)); - cput(0); - WPUT(shndx); - VPUT(addr); - VPUT(size); } void -asmelfsym64(void) +putelfsym(Sym *x, char *s, int t, vlong addr, vlong size, int ver, Sym *go) { - genasmsym(putelfsym64); -} + int bind, type, shndx, off; -void -putelfsym32(Sym *x, char *s, int t, vlong addr, vlong size, int ver, Sym *go) -{ - int bind, type, shndx, stroff; - - bind = STB_GLOBAL; switch(t) { default: return; @@ -113,27 +99,27 @@ putelfsym32(Sym *x, char *s, int t, vlong addr, vlong size, int ver, Sym *go) break; case 'D': type = STT_OBJECT; - shndx = elftextsh + 1; + if((x->type&~SSUB) == SRODATA) + shndx = elftextsh + 1; + else + shndx = elftextsh + 2; break; case 'B': type = STT_OBJECT; - shndx = elftextsh + 2; + shndx = elftextsh + 3; break; } - - stroff = putelfstr(s); - LPUT(stroff); // string - LPUT(addr); - LPUT(size); - cput((bind<<4)|(type&0xF)); - cput(0); - WPUT(shndx); + bind = ver ? STB_LOCAL : STB_GLOBAL; + off = putelfstr(s); + putelfsyment(off, addr, size, (bind<<4)|(type&0xf), shndx); } void -asmelfsym32(void) +asmelfsym(void) { - genasmsym(putelfsym32); + // the first symbol entry is reserved + putelfsyment(0, 0, 0, (STB_LOCAL<<4)|STT_NOTYPE, 0); + genasmsym(putelfsym); } void diff --git a/src/cmd/prof/gopprof b/src/cmd/prof/gopprof index 8863fc623..be5f84e9e 100755 --- a/src/cmd/prof/gopprof +++ b/src/cmd/prof/gopprof @@ -150,7 +150,8 @@ pprof [options] <profile> The /<service> can be $HEAP_PAGE, $PROFILE_PAGE, /pprof/pmuprofile, $GROWTH_PAGE, $CONTENTION_PAGE, /pprof/wall, or /pprof/filteredprofile. - For instance: "pprof http://myserver.com:80$HEAP_PAGE". + For instance: + pprof http://myserver.com:80$HEAP_PAGE If /<service> is omitted, the service defaults to $PROFILE_PAGE (cpu profiling). pprof --symbols <program> Maps addresses to symbol names. In this mode, stdin should be a @@ -532,7 +533,7 @@ sub Init() { ConfigureObjTools($main::prog) } - # Break the opt_list_prefix into the prefix_list array + # Break the opt_lib_prefix into the prefix_list array @prefix_list = split (',', $main::opt_lib_prefix); # Remove trailing / from the prefixes, in the list to prevent @@ -626,7 +627,7 @@ sub Main() { if ($main::opt_disasm) { PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm, $total); } elsif ($main::opt_list) { - PrintListing($libs, $flat, $cumulative, $main::opt_list); + PrintListing($total, $libs, $flat, $cumulative, $main::opt_list, 0); } elsif ($main::opt_text) { # Make sure the output is empty when have nothing to report # (only matters when --heapcheck is given but we must be @@ -814,7 +815,7 @@ sub InteractiveCommand { my $ignore; ($routine, $ignore) = ParseInteractiveArgs($3); - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles @@ -841,21 +842,22 @@ sub InteractiveCommand { return 1; } - if (m/^\s*list\s*(.+)/) { + if (m/^\s*(web)?list\s*(.+)/) { + my $html = (defined($1) && ($1 eq "web")); $main::opt_list = 1; my $routine; my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($1); + ($routine, $ignore) = ParseInteractiveArgs($2); - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles my $flat = FlatProfile($reduced); my $cumulative = CumulativeProfile($reduced); - PrintListing($libs, $flat, $cumulative, $routine); + PrintListing($total, $libs, $flat, $cumulative, $routine, $html); return 1; } if (m/^\s*disasm\s*(.+)/) { @@ -866,7 +868,7 @@ sub InteractiveCommand { ($routine, $ignore) = ParseInteractiveArgs($1); # Process current profile to account for various settings - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles @@ -890,7 +892,7 @@ sub InteractiveCommand { ($focus, $ignore) = ParseInteractiveArgs($2); # Process current profile to account for various settings - my $profile = ProcessProfile($orig_profile, $symbols, $focus, $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, $focus, $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles @@ -916,6 +918,7 @@ sub InteractiveCommand { sub ProcessProfile { + my $total_count = shift; my $orig_profile = shift; my $symbols = shift; my $focus = shift; @@ -923,7 +926,6 @@ sub ProcessProfile { # Process current profile to account for various settings my $profile = $orig_profile; - my $total_count = TotalProfile($profile); printf("Total: %s %s\n", Unparse($total_count), Units()); if ($focus ne '') { $profile = FocusProfile($symbols, $profile, $focus); @@ -970,6 +972,11 @@ Commands: list [routine_regexp] [-ignore1] [-ignore2] Show source listing of routines whose names match "routine_regexp" + weblist [routine_regexp] [-ignore1] [-ignore2] + Displays a source listing of routines whose names match "routine_regexp" + in a web browser. You can click on source lines to view the + corresponding disassembly. + top [--cum] [-ignore1] [-ignore2] top20 [--cum] [-ignore1] [-ignore2] top37 [--cum] [-ignore1] [-ignore2] @@ -1144,7 +1151,7 @@ sub PrintText { $sym); } $lines++; - last if ($line_limit >= 0 && $lines > $line_limit); + last if ($line_limit >= 0 && $lines >= $line_limit); } } @@ -1291,11 +1298,32 @@ sub ByName { # Print source-listing for all all routines that match $main::opt_list sub PrintListing { + my $total = shift; my $libs = shift; my $flat = shift; my $cumulative = shift; my $list_opts = shift; - + my $html = shift; + + my $output = \*STDOUT; + my $fname = ""; + + + if ($html) { + # Arrange to write the output to a temporary file + $fname = TempName($main::next_tmpfile, "html"); + $main::next_tmpfile++; + if (!open(TEMP, ">$fname")) { + print STDERR "$fname: $!\n"; + return; + } + $output = \*TEMP; + print $output HtmlListingHeader(); + printf $output ("<div class=\"legend\">%s<br>Total: %s %s</div>\n", + $main::prog, Unparse($total), Units()); + } + + my $listed = 0; foreach my $lib (@{$libs}) { my $symbol_table = GetProcedureBoundaries($lib->[0], $list_opts); my $offset = AddressSub($lib->[1], $lib->[3]); @@ -1307,15 +1335,98 @@ sub PrintListing { my $addr = AddressAdd($start_addr, $offset); for (my $i = 0; $i < $length; $i++) { if (defined($cumulative->{$addr})) { - PrintSource($lib->[0], $offset, - $routine, $flat, $cumulative, - $start_addr, $end_addr); + $listed += PrintSource( + $lib->[0], $offset, + $routine, $flat, $cumulative, + $start_addr, $end_addr, + $html, + $output); last; } $addr = AddressInc($addr); } } } + + if ($html) { + if ($listed > 0) { + print $output HtmlListingFooter(); + close($output); + RunWeb($fname); + } else { + close($output); + unlink($fname); + } + } +} + +sub HtmlListingHeader { + return <<'EOF'; +<DOCTYPE html> +<html> +<head> +<title>Pprof listing</title> +<style type="text/css"> +body { + font-family: sans-serif; +} +h1 { + font-size: 1.5em; + margin-bottom: 4px; +} +.legend { + font-size: 1.25em; +} +.line { + color: #aaaaaa; +} +.livesrc { + color: #0000ff; + cursor: pointer; +} +.livesrc:hover { + background-color: #cccccc; +} +.asm { + color: #888888; + display: none; +} +</style> +<script type="text/javascript"> +function pprof_toggle_asm(e) { + var target; + if (!e) e = window.event; + if (e.target) target = e.target; + else if (e.srcElement) target = e.srcElement; + + if (target && target.className == "livesrc") { + var asm = target.nextSibling; + if (asm && asm.className == "asm") { + asm.style.display = (asm.style.display == "block" ? "none" : "block"); + e.preventDefault(); + return false; + } + } +} +</script> +</head> +<body> +EOF +} + +sub HtmlListingFooter { + return <<'EOF'; +</body> +</html> +EOF +} + +sub HtmlEscape { + my $text = shift; + $text =~ s/&/&/g; + $text =~ s/</</g; + $text =~ s/>/>/g; + return $text; } # Returns the indentation of the line, if it has any non-whitespace @@ -1338,6 +1449,8 @@ sub PrintSource { my $cumulative = shift; my $start_addr = shift; my $end_addr = shift; + my $html = shift; + my $output = shift; # Disassemble all instructions (just to get line numbers) my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); @@ -1353,7 +1466,7 @@ sub PrintSource { } if (!defined($filename)) { print STDERR "no filename found in $routine\n"; - return; + return 0; } # Hack 2: assume that the largest line number from $filename is the @@ -1386,7 +1499,7 @@ sub PrintSource { { if (!open(FILE, "<$filename")) { print STDERR "$filename: $!\n"; - return; + return 0; } my $l = 0; my $first_indentation = -1; @@ -1414,12 +1527,21 @@ sub PrintSource { # Assign all samples to the range $firstline,$lastline, # Hack 4: If an instruction does not occur in the range, its samples # are moved to the next instruction that occurs in the range. - my $samples1 = {}; - my $samples2 = {}; - my $running1 = 0; # Unassigned flat counts - my $running2 = 0; # Unassigned cumulative counts - my $total1 = 0; # Total flat counts - my $total2 = 0; # Total cumulative counts + my $samples1 = {}; # Map from line number to flat count + my $samples2 = {}; # Map from line number to cumulative count + my $running1 = 0; # Unassigned flat counts + my $running2 = 0; # Unassigned cumulative counts + my $total1 = 0; # Total flat counts + my $total2 = 0; # Total cumulative counts + my %disasm = (); # Map from line number to disassembly + my $running_disasm = ""; # Unassigned disassembly + my $skip_marker = "---\n"; + if ($html) { + $skip_marker = ""; + for (my $l = $firstline; $l <= $lastline; $l++) { + $disasm{$l} = ""; + } + } foreach my $e (@instructions) { # Add up counts for all address that fall inside this instruction my $c1 = 0; @@ -1428,6 +1550,15 @@ sub PrintSource { $c1 += GetEntry($flat, $a); $c2 += GetEntry($cumulative, $a); } + + if ($html) { + $running_disasm .= sprintf(" %6s %6s \t\t%8s: %s\n", + HtmlPrintNumber($c1), + HtmlPrintNumber($c2), + $e->[0], + CleanDisassembly($e->[3])); + } + $running1 += $c1; $running2 += $c2; $total1 += $c1; @@ -1442,6 +1573,10 @@ sub PrintSource { AddEntry($samples2, $line, $running2); $running1 = 0; $running2 = 0; + if ($html) { + $disasm{$line} .= $running_disasm; + $running_disasm = ''; + } } } @@ -1449,16 +1584,28 @@ sub PrintSource { AddEntry($samples1, $lastline, $running1); AddEntry($samples2, $lastline, $running2); - printf("ROUTINE ====================== %s in %s\n" . - "%6s %6s Total %s (flat / cumulative)\n", - ShortFunctionName($routine), - $filename, - Units(), - Unparse($total1), - Unparse($total2)); + if ($html) { + printf $output ( + "<h1>%s</h1>%s\n<pre onClick=\"pprof_toggle_asm()\">\n" . + "Total:%6s %6s (flat / cumulative %s)\n", + HtmlEscape(ShortFunctionName($routine)), + HtmlEscape($filename), + Unparse($total1), + Unparse($total2), + Units()); + } else { + printf $output ( + "ROUTINE ====================== %s in %s\n" . + "%6s %6s Total %s (flat / cumulative)\n", + ShortFunctionName($routine), + $filename, + Unparse($total1), + Unparse($total2), + Units()); + } if (!open(FILE, "<$filename")) { print STDERR "$filename: $!\n"; - return; + return 0; } my $l = 0; while (<FILE>) { @@ -1468,16 +1615,47 @@ sub PrintSource { (($l <= $oldlastline + 5) || ($l <= $lastline))) { chop; my $text = $_; - if ($l == $firstline) { printf("---\n"); } - printf("%6s %6s %4d: %s\n", - UnparseAlt(GetEntry($samples1, $l)), - UnparseAlt(GetEntry($samples2, $l)), - $l, - $text); - if ($l == $lastline) { printf("---\n"); } + if ($l == $firstline) { print $output $skip_marker; } + my $n1 = GetEntry($samples1, $l); + my $n2 = GetEntry($samples2, $l); + if ($html) { + my $dis = $disasm{$l}; + if (!defined($dis) || $n1 + $n2 == 0) { + # No samples/disassembly for this source line + printf $output ( + "<span class=\"line\">%5d</span> " . + "<span class=\"deadsrc\">%6s %6s %s</span>\n", + $l, + HtmlPrintNumber($n1), + HtmlPrintNumber($n2), + HtmlEscape($text)); + } else { + printf $output ( + "<span class=\"line\">%5d</span> " . + "<span class=\"livesrc\">%6s %6s %s</span>" . + "<span class=\"asm\">%s</span>\n", + $l, + HtmlPrintNumber($n1), + HtmlPrintNumber($n2), + HtmlEscape($text), + HtmlEscape($dis)); + } + } else { + printf $output( + "%6s %6s %4d: %s\n", + UnparseAlt($n1), + UnparseAlt($n2), + $l, + $text); + } + if ($l == $lastline) { print $output $skip_marker; } }; } close(FILE); + if ($html) { + print $output "</pre>\n"; + } + return 1; } # Return the source line for the specified file/linenumber. @@ -1625,16 +1803,11 @@ sub PrintDisassembledFunction { $address =~ s/^0x//; $address =~ s/^0*//; - # Trim symbols - my $d = $e->[3]; - while ($d =~ s/\([^()%]*\)(\s*const)?//g) { } # Argument types, not (%rax) - while ($d =~ s/(\w+)<[^<>]*>/$1/g) { } # Remove template arguments - printf("%6s %6s %8s: %6s\n", UnparseAlt($flat_count[$x]), UnparseAlt($cum_count[$x]), $address, - $d); + CleanDisassembly($e->[3])); } } } @@ -2254,6 +2427,16 @@ sub UnparseAlt { } } +# Alternate pretty-printed form: 0 maps to "" +sub HtmlPrintNumber { + my $num = shift; + if ($num == 0) { + return ""; + } else { + return Unparse($num); + } +} + # Return output units sub Units { if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { @@ -2415,6 +2598,8 @@ sub RemoveUninterestingFrames { 'copyin', 'gostring', 'gostringsize', + 'growslice1', + 'appendslice1', 'hash_init', 'hash_subtable_new', 'hash_conv', @@ -2422,6 +2607,8 @@ sub RemoveUninterestingFrames { 'hash_insert_internal', 'hash_insert', 'mapassign', + 'runtime.mapassign', + 'runtime.appendslice', 'runtime.mapassign1', 'makechan', 'makemap', @@ -2433,11 +2620,13 @@ sub RemoveUninterestingFrames { 'unsafe.New', 'runtime.mallocgc', 'runtime.catstring', + 'runtime.growslice', 'runtime.ifaceT2E', 'runtime.ifaceT2I', 'runtime.makechan', 'runtime.makechan_c', 'runtime.makemap', + 'runtime.makemap_c', 'runtime.makeslice', 'runtime.mal', 'runtime.slicebytetostring', @@ -4302,6 +4491,14 @@ sub ShortFunctionName { return $function; } +# Trim overly long symbols found in disassembler output +sub CleanDisassembly { + my $d = shift; + while ($d =~ s/\([^()%]*\)(\s*const)?//g) { } # Argument types, not (%rax) + while ($d =~ s/(\w+)<[^<>]*>/$1/g) { } # Remove template arguments + return $d; +} + ##### Miscellaneous ##### # Find the right versions of the above object tools to use. The diff --git a/src/libmach/8db.c b/src/libmach/8db.c index 80aa4fe69..5a195baf8 100644 --- a/src/libmach/8db.c +++ b/src/libmach/8db.c @@ -62,32 +62,32 @@ static char LESSSTACK[] = "sys·lessstack"; static char MORESTACK[] = "sys·morestack"; static char *excname[] = { -[0] "divide error", -[1] "debug exception", -[4] "overflow", -[5] "bounds check", -[6] "invalid opcode", -[7] "math coprocessor emulation", -[8] "double fault", -[9] "math coprocessor overrun", -[10] "invalid TSS", -[11] "segment not present", -[12] "stack exception", -[13] "general protection violation", -[14] "page fault", -[16] "math coprocessor error", -[17] "alignment check", -[18] "machine check", -[19] "floating-point exception", -[24] "clock", -[25] "keyboard", -[27] "modem status", -[28] "serial line status", -[30] "floppy disk", -[36] "mouse", -[37] "math coprocessor", -[38] "hard disk", -[64] "system call", +[0] = "divide error", +[1] = "debug exception", +[4] = "overflow", +[5] = "bounds check", +[6] = "invalid opcode", +[7] = "math coprocessor emulation", +[8] = "double fault", +[9] = "math coprocessor overrun", +[10] = "invalid TSS", +[11] = "segment not present", +[12] = "stack exception", +[13] = "general protection violation", +[14] = "page fault", +[16] = "math coprocessor error", +[17] = "alignment check", +[18] = "machine check", +[19] = "floating-point exception", +[24] = "clock", +[25] = "keyboard", +[27] = "modem status", +[28] = "serial line status", +[30] = "floppy disk", +[36] = "mouse", +[37] = "math coprocessor", +[38] = "hard disk", +[64] = "system call", }; Machdata i386mach = @@ -436,28 +436,28 @@ enum { static Optable optab0F00[8]= { -[0x00] 0,0, "MOVW LDT,%e", -[0x01] 0,0, "MOVW TR,%e", -[0x02] 0,0, "MOVW %e,LDT", -[0x03] 0,0, "MOVW %e,TR", -[0x04] 0,0, "VERR %e", -[0x05] 0,0, "VERW %e", +[0x00] = { 0,0, "MOVW LDT,%e" }, +[0x01] = { 0,0, "MOVW TR,%e" }, +[0x02] = { 0,0, "MOVW %e,LDT" }, +[0x03] = { 0,0, "MOVW %e,TR" }, +[0x04] = { 0,0, "VERR %e" }, +[0x05] = { 0,0, "VERW %e" }, }; static Optable optab0F01[8]= { -[0x00] 0,0, "MOVL GDTR,%e", -[0x01] 0,0, "MOVL IDTR,%e", -[0x02] 0,0, "MOVL %e,GDTR", -[0x03] 0,0, "MOVL %e,IDTR", -[0x04] 0,0, "MOVW MSW,%e", /* word */ -[0x06] 0,0, "MOVW %e,MSW", /* word */ -[0x07] 0,0, "INVLPG %e", /* or SWAPGS */ +[0x00] = { 0,0, "MOVL GDTR,%e" }, +[0x01] = { 0,0, "MOVL IDTR,%e" }, +[0x02] = { 0,0, "MOVL %e,GDTR" }, +[0x03] = { 0,0, "MOVL %e,IDTR" }, +[0x04] = { 0,0, "MOVW MSW,%e" }, /* word */ +[0x06] = { 0,0, "MOVW %e,MSW" }, /* word */ +[0x07] = { 0,0, "INVLPG %e" }, /* or SWAPGS */ }; static Optable optab0F01F8[1]= { -[0x00] 0,0, "SWAPGS", +[0x00] = { 0,0, "SWAPGS" }, }; /* 0F71 */ @@ -466,13 +466,13 @@ static Optable optab0F01F8[1]= static Optable optab0FAE[8]= { -[0x00] 0,0, "FXSAVE %e", -[0x01] 0,0, "FXRSTOR %e", -[0x02] 0,0, "LDMXCSR %e", -[0x03] 0,0, "STMXCSR %e", -[0x05] 0,0, "LFENCE", -[0x06] 0,0, "MFENCE", -[0x07] 0,0, "SFENCE", +[0x00] = { 0,0, "FXSAVE %e" }, +[0x01] = { 0,0, "FXRSTOR %e" }, +[0x02] = { 0,0, "LDMXCSR %e" }, +[0x03] = { 0,0, "STMXCSR %e" }, +[0x05] = { 0,0, "LFENCE" }, +[0x06] = { 0,0, "MFENCE" }, +[0x07] = { 0,0, "SFENCE" }, }; /* 0F18 */ @@ -480,455 +480,455 @@ static Optable optab0FAE[8]= static Optable optab0FBA[8]= { -[0x04] Ib,0, "BT%S %i,%e", -[0x05] Ib,0, "BTS%S %i,%e", -[0x06] Ib,0, "BTR%S %i,%e", -[0x07] Ib,0, "BTC%S %i,%e", +[0x04] = { Ib,0, "BT%S %i,%e" }, +[0x05] = { Ib,0, "BTS%S %i,%e" }, +[0x06] = { Ib,0, "BTR%S %i,%e" }, +[0x07] = { Ib,0, "BTC%S %i,%e" }, }; static Optable optab0F0F[256]= { -[0x0c] 0,0, "PI2FW %m,%M", -[0x0d] 0,0, "PI2L %m,%M", -[0x1c] 0,0, "PF2IW %m,%M", -[0x1d] 0,0, "PF2IL %m,%M", -[0x8a] 0,0, "PFNACC %m,%M", -[0x8e] 0,0, "PFPNACC %m,%M", -[0x90] 0,0, "PFCMPGE %m,%M", -[0x94] 0,0, "PFMIN %m,%M", -[0x96] 0,0, "PFRCP %m,%M", -[0x97] 0,0, "PFRSQRT %m,%M", -[0x9a] 0,0, "PFSUB %m,%M", -[0x9e] 0,0, "PFADD %m,%M", -[0xa0] 0,0, "PFCMPGT %m,%M", -[0xa4] 0,0, "PFMAX %m,%M", -[0xa6] 0,0, "PFRCPIT1 %m,%M", -[0xa7] 0,0, "PFRSQIT1 %m,%M", -[0xaa] 0,0, "PFSUBR %m,%M", -[0xae] 0,0, "PFACC %m,%M", -[0xb0] 0,0, "PFCMPEQ %m,%M", -[0xb4] 0,0, "PFMUL %m,%M", -[0xb6] 0,0, "PFRCPI2T %m,%M", -[0xb7] 0,0, "PMULHRW %m,%M", -[0xbb] 0,0, "PSWAPL %m,%M", +[0x0c] = { 0,0, "PI2FW %m,%M" }, +[0x0d] = { 0,0, "PI2L %m,%M" }, +[0x1c] = { 0,0, "PF2IW %m,%M" }, +[0x1d] = { 0,0, "PF2IL %m,%M" }, +[0x8a] = { 0,0, "PFNACC %m,%M" }, +[0x8e] = { 0,0, "PFPNACC %m,%M" }, +[0x90] = { 0,0, "PFCMPGE %m,%M" }, +[0x94] = { 0,0, "PFMIN %m,%M" }, +[0x96] = { 0,0, "PFRCP %m,%M" }, +[0x97] = { 0,0, "PFRSQRT %m,%M" }, +[0x9a] = { 0,0, "PFSUB %m,%M" }, +[0x9e] = { 0,0, "PFADD %m,%M" }, +[0xa0] = { 0,0, "PFCMPGT %m,%M" }, +[0xa4] = { 0,0, "PFMAX %m,%M" }, +[0xa6] = { 0,0, "PFRCPIT1 %m,%M" }, +[0xa7] = { 0,0, "PFRSQIT1 %m,%M" }, +[0xaa] = { 0,0, "PFSUBR %m,%M" }, +[0xae] = { 0,0, "PFACC %m,%M" }, +[0xb0] = { 0,0, "PFCMPEQ %m,%M" }, +[0xb4] = { 0,0, "PFMUL %m,%M" }, +[0xb6] = { 0,0, "PFRCPI2T %m,%M" }, +[0xb7] = { 0,0, "PMULHRW %m,%M" }, +[0xbb] = { 0,0, "PSWAPL %m,%M" }, }; static Optable optab0FC7[8]= { -[0x01] 0,0, "CMPXCHG8B %e", +[0x01] = { 0,0, "CMPXCHG8B %e" }, }; static Optable optab660F71[8]= { -[0x02] Ib,0, "PSRLW %i,%X", -[0x04] Ib,0, "PSRAW %i,%X", -[0x06] Ib,0, "PSLLW %i,%X", +[0x02] = { Ib,0, "PSRLW %i,%X" }, +[0x04] = { Ib,0, "PSRAW %i,%X" }, +[0x06] = { Ib,0, "PSLLW %i,%X" }, }; static Optable optab660F72[8]= { -[0x02] Ib,0, "PSRLL %i,%X", -[0x04] Ib,0, "PSRAL %i,%X", -[0x06] Ib,0, "PSLLL %i,%X", +[0x02] = { Ib,0, "PSRLL %i,%X" }, +[0x04] = { Ib,0, "PSRAL %i,%X" }, +[0x06] = { Ib,0, "PSLLL %i,%X" }, }; static Optable optab660F73[8]= { -[0x02] Ib,0, "PSRLQ %i,%X", -[0x03] Ib,0, "PSRLO %i,%X", -[0x06] Ib,0, "PSLLQ %i,%X", -[0x07] Ib,0, "PSLLO %i,%X", +[0x02] = { Ib,0, "PSRLQ %i,%X" }, +[0x03] = { Ib,0, "PSRLO %i,%X" }, +[0x06] = { Ib,0, "PSLLQ %i,%X" }, +[0x07] = { Ib,0, "PSLLO %i,%X" }, }; static Optable optab660F[256]= { -[0x2B] RM,0, "MOVNTPD %x,%e", -[0x2E] RM,0, "UCOMISD %x,%X", -[0x2F] RM,0, "COMISD %x,%X", -[0x5A] RM,0, "CVTPD2PS %x,%X", -[0x5B] RM,0, "CVTPS2PL %x,%X", -[0x6A] RM,0, "PUNPCKHLQ %x,%X", -[0x6B] RM,0, "PACKSSLW %x,%X", -[0x6C] RM,0, "PUNPCKLQDQ %x,%X", -[0x6D] RM,0, "PUNPCKHQDQ %x,%X", -[0x6E] RM,0, "MOV%S %e,%X", -[0x6F] RM,0, "MOVO %x,%X", /* MOVDQA */ -[0x70] RM,Ib, "PSHUFL %i,%x,%X", -[0x71] RMOP,0, optab660F71, -[0x72] RMOP,0, optab660F72, -[0x73] RMOP,0, optab660F73, -[0x7E] RM,0, "MOV%S %X,%e", -[0x7F] RM,0, "MOVO %X,%x", -[0xC4] RM,Ib, "PINSRW %i,%e,%X", -[0xC5] RMR,Ib, "PEXTRW %i,%X,%e", -[0xD4] RM,0, "PADDQ %x,%X", -[0xD5] RM,0, "PMULLW %x,%X", -[0xD6] RM,0, "MOVQ %X,%x", -[0xE6] RM,0, "CVTTPD2PL %x,%X", -[0xE7] RM,0, "MOVNTO %X,%e", -[0xF7] RM,0, "MASKMOVOU %x,%X", +[0x2B] = { RM,0, "MOVNTPD %x,%e" }, +[0x2E] = { RM,0, "UCOMISD %x,%X" }, +[0x2F] = { RM,0, "COMISD %x,%X" }, +[0x5A] = { RM,0, "CVTPD2PS %x,%X" }, +[0x5B] = { RM,0, "CVTPS2PL %x,%X" }, +[0x6A] = { RM,0, "PUNPCKHLQ %x,%X" }, +[0x6B] = { RM,0, "PACKSSLW %x,%X" }, +[0x6C] = { RM,0, "PUNPCKLQDQ %x,%X" }, +[0x6D] = { RM,0, "PUNPCKHQDQ %x,%X" }, +[0x6E] = { RM,0, "MOV%S %e,%X" }, +[0x6F] = { RM,0, "MOVO %x,%X" }, /* MOVDQA */ +[0x70] = { RM,Ib, "PSHUFL %i,%x,%X" }, +[0x71] = { RMOP,0, optab660F71 }, +[0x72] = { RMOP,0, optab660F72 }, +[0x73] = { RMOP,0, optab660F73 }, +[0x7E] = { RM,0, "MOV%S %X,%e" }, +[0x7F] = { RM,0, "MOVO %X,%x" }, +[0xC4] = { RM,Ib, "PINSRW %i,%e,%X" }, +[0xC5] = { RMR,Ib, "PEXTRW %i,%X,%e" }, +[0xD4] = { RM,0, "PADDQ %x,%X" }, +[0xD5] = { RM,0, "PMULLW %x,%X" }, +[0xD6] = { RM,0, "MOVQ %X,%x" }, +[0xE6] = { RM,0, "CVTTPD2PL %x,%X" }, +[0xE7] = { RM,0, "MOVNTO %X,%e" }, +[0xF7] = { RM,0, "MASKMOVOU %x,%X" }, }; static Optable optabF20F[256]= { -[0x10] RM,0, "MOVSD %x,%X", -[0x11] RM,0, "MOVSD %X,%x", -[0x2A] RM,0, "CVTS%S2SD %e,%X", -[0x2C] RM,0, "CVTTSD2S%S %x,%r", -[0x2D] RM,0, "CVTSD2S%S %x,%r", -[0x5A] RM,0, "CVTSD2SS %x,%X", -[0x6F] RM,0, "MOVOU %x,%X", -[0x70] RM,Ib, "PSHUFLW %i,%x,%X", -[0x7F] RM,0, "MOVOU %X,%x", -[0xD6] RM,0, "MOVQOZX %M,%X", -[0xE6] RM,0, "CVTPD2PL %x,%X", +[0x10] = { RM,0, "MOVSD %x,%X" }, +[0x11] = { RM,0, "MOVSD %X,%x" }, +[0x2A] = { RM,0, "CVTS%S2SD %e,%X" }, +[0x2C] = { RM,0, "CVTTSD2S%S %x,%r" }, +[0x2D] = { RM,0, "CVTSD2S%S %x,%r" }, +[0x5A] = { RM,0, "CVTSD2SS %x,%X" }, +[0x6F] = { RM,0, "MOVOU %x,%X" }, +[0x70] = { RM,Ib, "PSHUFLW %i,%x,%X" }, +[0x7F] = { RM,0, "MOVOU %X,%x" }, +[0xD6] = { RM,0, "MOVQOZX %M,%X" }, +[0xE6] = { RM,0, "CVTPD2PL %x,%X" }, }; static Optable optabF30F[256]= { -[0x10] RM,0, "MOVSS %x,%X", -[0x11] RM,0, "MOVSS %X,%x", -[0x2A] RM,0, "CVTS%S2SS %e,%X", -[0x2C] RM,0, "CVTTSS2S%S %x,%r", -[0x2D] RM,0, "CVTSS2S%S %x,%r", -[0x5A] RM,0, "CVTSS2SD %x,%X", -[0x5B] RM,0, "CVTTPS2PL %x,%X", -[0x6F] RM,0, "MOVOU %x,%X", -[0x70] RM,Ib, "PSHUFHW %i,%x,%X", -[0x7E] RM,0, "MOVQOZX %x,%X", -[0x7F] RM,0, "MOVOU %X,%x", -[0xD6] RM,0, "MOVQOZX %m*,%X", -[0xE6] RM,0, "CVTPL2PD %x,%X", +[0x10] = { RM,0, "MOVSS %x,%X" }, +[0x11] = { RM,0, "MOVSS %X,%x" }, +[0x2A] = { RM,0, "CVTS%S2SS %e,%X" }, +[0x2C] = { RM,0, "CVTTSS2S%S %x,%r" }, +[0x2D] = { RM,0, "CVTSS2S%S %x,%r" }, +[0x5A] = { RM,0, "CVTSS2SD %x,%X" }, +[0x5B] = { RM,0, "CVTTPS2PL %x,%X" }, +[0x6F] = { RM,0, "MOVOU %x,%X" }, +[0x70] = { RM,Ib, "PSHUFHW %i,%x,%X" }, +[0x7E] = { RM,0, "MOVQOZX %x,%X" }, +[0x7F] = { RM,0, "MOVOU %X,%x" }, +[0xD6] = { RM,0, "MOVQOZX %m*,%X" }, +[0xE6] = { RM,0, "CVTPL2PD %x,%X" }, }; static Optable optab0F[256]= { -[0x00] RMOP,0, optab0F00, -[0x01] RMOP,0, optab0F01, -[0x02] RM,0, "LAR %e,%r", -[0x03] RM,0, "LSL %e,%r", -[0x05] 0,0, "SYSCALL", -[0x06] 0,0, "CLTS", -[0x07] 0,0, "SYSRET", -[0x08] 0,0, "INVD", -[0x09] 0,0, "WBINVD", -[0x0B] 0,0, "UD2", -[0x0F] RM,AUX, optab0F0F, /* 3DNow! */ -[0x10] RM,0, "MOVU%s %x,%X", -[0x11] RM,0, "MOVU%s %X,%x", -[0x12] RM,0, "MOV[H]L%s %x,%X", /* TO DO: H if source is XMM */ -[0x13] RM,0, "MOVL%s %X,%e", -[0x14] RM,0, "UNPCKL%s %x,%X", -[0x15] RM,0, "UNPCKH%s %x,%X", -[0x16] RM,0, "MOV[L]H%s %x,%X", /* TO DO: L if source is XMM */ -[0x17] RM,0, "MOVH%s %X,%x", -[0x20] RMR,0, "MOVL %C,%e", -[0x21] RMR,0, "MOVL %D,%e", -[0x22] RMR,0, "MOVL %e,%C", -[0x23] RMR,0, "MOVL %e,%D", -[0x24] RMR,0, "MOVL %T,%e", -[0x26] RMR,0, "MOVL %e,%T", -[0x28] RM,0, "MOVA%s %x,%X", -[0x29] RM,0, "MOVA%s %X,%x", -[0x2A] RM,0, "CVTPL2%s %m*,%X", -[0x2B] RM,0, "MOVNT%s %X,%e", -[0x2C] RM,0, "CVTT%s2PL %x,%M", -[0x2D] RM,0, "CVT%s2PL %x,%M", -[0x2E] RM,0, "UCOMISS %x,%X", -[0x2F] RM,0, "COMISS %x,%X", -[0x30] 0,0, "WRMSR", -[0x31] 0,0, "RDTSC", -[0x32] 0,0, "RDMSR", -[0x33] 0,0, "RDPMC", -[0x42] RM,0, "CMOVC %e,%r", /* CF */ -[0x43] RM,0, "CMOVNC %e,%r", /* ¬ CF */ -[0x44] RM,0, "CMOVZ %e,%r", /* ZF */ -[0x45] RM,0, "CMOVNZ %e,%r", /* ¬ ZF */ -[0x46] RM,0, "CMOVBE %e,%r", /* CF ∨ ZF */ -[0x47] RM,0, "CMOVA %e,%r", /* ¬CF ∧ ¬ZF */ -[0x48] RM,0, "CMOVS %e,%r", /* SF */ -[0x49] RM,0, "CMOVNS %e,%r", /* ¬ SF */ -[0x4A] RM,0, "CMOVP %e,%r", /* PF */ -[0x4B] RM,0, "CMOVNP %e,%r", /* ¬ PF */ -[0x4C] RM,0, "CMOVLT %e,%r", /* LT ≡ OF ≠ SF */ -[0x4D] RM,0, "CMOVGE %e,%r", /* GE ≡ ZF ∨ SF */ -[0x4E] RM,0, "CMOVLE %e,%r", /* LE ≡ ZF ∨ LT */ -[0x4F] RM,0, "CMOVGT %e,%r", /* GT ≡ ¬ZF ∧ GE */ -[0x50] RM,0, "MOVMSK%s %X,%r", /* TO DO: check */ -[0x51] RM,0, "SQRT%s %x,%X", -[0x52] RM,0, "RSQRT%s %x,%X", -[0x53] RM,0, "RCP%s %x,%X", -[0x54] RM,0, "AND%s %x,%X", -[0x55] RM,0, "ANDN%s %x,%X", -[0x56] RM,0, "OR%s %x,%X", /* TO DO: S/D */ -[0x57] RM,0, "XOR%s %x,%X", /* S/D */ -[0x58] RM,0, "ADD%s %x,%X", /* S/P S/D */ -[0x59] RM,0, "MUL%s %x,%X", -[0x5A] RM,0, "CVTPS2PD %x,%X", -[0x5B] RM,0, "CVTPL2PS %x,%X", -[0x5C] RM,0, "SUB%s %x,%X", -[0x5D] RM,0, "MIN%s %x,%X", -[0x5E] RM,0, "DIV%s %x,%X", /* TO DO: S/P S/D */ -[0x5F] RM,0, "MAX%s %x,%X", -[0x60] RM,0, "PUNPCKLBW %m,%M", -[0x61] RM,0, "PUNPCKLWL %m,%M", -[0x62] RM,0, "PUNPCKLLQ %m,%M", -[0x63] RM,0, "PACKSSWB %m,%M", -[0x64] RM,0, "PCMPGTB %m,%M", -[0x65] RM,0, "PCMPGTW %m,%M", -[0x66] RM,0, "PCMPGTL %m,%M", -[0x67] RM,0, "PACKUSWB %m,%M", -[0x68] RM,0, "PUNPCKHBW %m,%M", -[0x69] RM,0, "PUNPCKHWL %m,%M", -[0x6A] RM,0, "PUNPCKHLQ %m,%M", -[0x6B] RM,0, "PACKSSLW %m,%M", -[0x6E] RM,0, "MOV%S %e,%M", -[0x6F] RM,0, "MOVQ %m,%M", -[0x70] RM,Ib, "PSHUFW %i,%m,%M", -[0x74] RM,0, "PCMPEQB %m,%M", -[0x75] RM,0, "PCMPEQW %m,%M", -[0x76] RM,0, "PCMPEQL %m,%M", -[0x7E] RM,0, "MOV%S %M,%e", -[0x7F] RM,0, "MOVQ %M,%m", -[0xAE] RMOP,0, optab0FAE, -[0xAA] 0,0, "RSM", -[0xB0] RM,0, "CMPXCHGB %r,%e", -[0xB1] RM,0, "CMPXCHG%S %r,%e", -[0xC0] RMB,0, "XADDB %r,%e", -[0xC1] RM,0, "XADD%S %r,%e", -[0xC2] RM,Ib, "CMP%s %x,%X,%#i", -[0xC3] RM,0, "MOVNTI%S %r,%e", -[0xC6] RM,Ib, "SHUF%s %i,%x,%X", -[0xC8] 0,0, "BSWAP AX", -[0xC9] 0,0, "BSWAP CX", -[0xCA] 0,0, "BSWAP DX", -[0xCB] 0,0, "BSWAP BX", -[0xCC] 0,0, "BSWAP SP", -[0xCD] 0,0, "BSWAP BP", -[0xCE] 0,0, "BSWAP SI", -[0xCF] 0,0, "BSWAP DI", -[0xD1] RM,0, "PSRLW %m,%M", -[0xD2] RM,0, "PSRLL %m,%M", -[0xD3] RM,0, "PSRLQ %m,%M", -[0xD5] RM,0, "PMULLW %m,%M", -[0xD6] RM,0, "MOVQOZX %m*,%X", -[0xD7] RM,0, "PMOVMSKB %m,%r", -[0xD8] RM,0, "PSUBUSB %m,%M", -[0xD9] RM,0, "PSUBUSW %m,%M", -[0xDA] RM,0, "PMINUB %m,%M", -[0xDB] RM,0, "PAND %m,%M", -[0xDC] RM,0, "PADDUSB %m,%M", -[0xDD] RM,0, "PADDUSW %m,%M", -[0xDE] RM,0, "PMAXUB %m,%M", -[0xDF] RM,0, "PANDN %m,%M", -[0xE0] RM,0, "PAVGB %m,%M", -[0xE1] RM,0, "PSRAW %m,%M", -[0xE2] RM,0, "PSRAL %m,%M", -[0xE3] RM,0, "PAVGW %m,%M", -[0xE4] RM,0, "PMULHUW %m,%M", -[0xE5] RM,0, "PMULHW %m,%M", -[0xE7] RM,0, "MOVNTQ %M,%e", -[0xE8] RM,0, "PSUBSB %m,%M", -[0xE9] RM,0, "PSUBSW %m,%M", -[0xEA] RM,0, "PMINSW %m,%M", -[0xEB] RM,0, "POR %m,%M", -[0xEC] RM,0, "PADDSB %m,%M", -[0xED] RM,0, "PADDSW %m,%M", -[0xEE] RM,0, "PMAXSW %m,%M", -[0xEF] RM,0, "PXOR %m,%M", -[0xF1] RM,0, "PSLLW %m,%M", -[0xF2] RM,0, "PSLLL %m,%M", -[0xF3] RM,0, "PSLLQ %m,%M", -[0xF4] RM,0, "PMULULQ %m,%M", -[0xF5] RM,0, "PMADDWL %m,%M", -[0xF6] RM,0, "PSADBW %m,%M", -[0xF7] RMR,0, "MASKMOVQ %m,%M", -[0xF8] RM,0, "PSUBB %m,%M", -[0xF9] RM,0, "PSUBW %m,%M", -[0xFA] RM,0, "PSUBL %m,%M", -[0xFC] RM,0, "PADDB %m,%M", -[0xFD] RM,0, "PADDW %m,%M", -[0xFE] RM,0, "PADDL %m,%M", - -[0x80] Iwds,0, "JOS %p", -[0x81] Iwds,0, "JOC %p", -[0x82] Iwds,0, "JCS %p", -[0x83] Iwds,0, "JCC %p", -[0x84] Iwds,0, "JEQ %p", -[0x85] Iwds,0, "JNE %p", -[0x86] Iwds,0, "JLS %p", -[0x87] Iwds,0, "JHI %p", -[0x88] Iwds,0, "JMI %p", -[0x89] Iwds,0, "JPL %p", -[0x8a] Iwds,0, "JPS %p", -[0x8b] Iwds,0, "JPC %p", -[0x8c] Iwds,0, "JLT %p", -[0x8d] Iwds,0, "JGE %p", -[0x8e] Iwds,0, "JLE %p", -[0x8f] Iwds,0, "JGT %p", -[0x90] RMB,0, "SETOS %e", -[0x91] RMB,0, "SETOC %e", -[0x92] RMB,0, "SETCS %e", -[0x93] RMB,0, "SETCC %e", -[0x94] RMB,0, "SETEQ %e", -[0x95] RMB,0, "SETNE %e", -[0x96] RMB,0, "SETLS %e", -[0x97] RMB,0, "SETHI %e", -[0x98] RMB,0, "SETMI %e", -[0x99] RMB,0, "SETPL %e", -[0x9a] RMB,0, "SETPS %e", -[0x9b] RMB,0, "SETPC %e", -[0x9c] RMB,0, "SETLT %e", -[0x9d] RMB,0, "SETGE %e", -[0x9e] RMB,0, "SETLE %e", -[0x9f] RMB,0, "SETGT %e", -[0xa0] 0,0, "PUSHL FS", -[0xa1] 0,0, "POPL FS", -[0xa2] 0,0, "CPUID", -[0xa3] RM,0, "BT%S %r,%e", -[0xa4] RM,Ib, "SHLD%S %r,%i,%e", -[0xa5] RM,0, "SHLD%S %r,CL,%e", -[0xa8] 0,0, "PUSHL GS", -[0xa9] 0,0, "POPL GS", -[0xab] RM,0, "BTS%S %r,%e", -[0xac] RM,Ib, "SHRD%S %r,%i,%e", -[0xad] RM,0, "SHRD%S %r,CL,%e", -[0xaf] RM,0, "IMUL%S %e,%r", -[0xb2] RMM,0, "LSS %e,%r", -[0xb3] RM,0, "BTR%S %r,%e", -[0xb4] RMM,0, "LFS %e,%r", -[0xb5] RMM,0, "LGS %e,%r", -[0xb6] RMB,0, "MOVBZX %e,%R", -[0xb7] RM,0, "MOVWZX %e,%R", -[0xba] RMOP,0, optab0FBA, -[0xbb] RM,0, "BTC%S %e,%r", -[0xbc] RM,0, "BSF%S %e,%r", -[0xbd] RM,0, "BSR%S %e,%r", -[0xbe] RMB,0, "MOVBSX %e,%R", -[0xbf] RM,0, "MOVWSX %e,%R", -[0xc7] RMOP,0, optab0FC7, +[0x00] = { RMOP,0, optab0F00 }, +[0x01] = { RMOP,0, optab0F01 }, +[0x02] = { RM,0, "LAR %e,%r" }, +[0x03] = { RM,0, "LSL %e,%r" }, +[0x05] = { 0,0, "SYSCALL" }, +[0x06] = { 0,0, "CLTS" }, +[0x07] = { 0,0, "SYSRET" }, +[0x08] = { 0,0, "INVD" }, +[0x09] = { 0,0, "WBINVD" }, +[0x0B] = { 0,0, "UD2" }, +[0x0F] = { RM,AUX, optab0F0F }, /* 3DNow! */ +[0x10] = { RM,0, "MOVU%s %x,%X" }, +[0x11] = { RM,0, "MOVU%s %X,%x" }, +[0x12] = { RM,0, "MOV[H]L%s %x,%X" }, /* TO DO: H if source is XMM */ +[0x13] = { RM,0, "MOVL%s %X,%e" }, +[0x14] = { RM,0, "UNPCKL%s %x,%X" }, +[0x15] = { RM,0, "UNPCKH%s %x,%X" }, +[0x16] = { RM,0, "MOV[L]H%s %x,%X" }, /* TO DO: L if source is XMM */ +[0x17] = { RM,0, "MOVH%s %X,%x" }, +[0x20] = { RMR,0, "MOVL %C,%e" }, +[0x21] = { RMR,0, "MOVL %D,%e" }, +[0x22] = { RMR,0, "MOVL %e,%C" }, +[0x23] = { RMR,0, "MOVL %e,%D" }, +[0x24] = { RMR,0, "MOVL %T,%e" }, +[0x26] = { RMR,0, "MOVL %e,%T" }, +[0x28] = { RM,0, "MOVA%s %x,%X" }, +[0x29] = { RM,0, "MOVA%s %X,%x" }, +[0x2A] = { RM,0, "CVTPL2%s %m*,%X" }, +[0x2B] = { RM,0, "MOVNT%s %X,%e" }, +[0x2C] = { RM,0, "CVTT%s2PL %x,%M" }, +[0x2D] = { RM,0, "CVT%s2PL %x,%M" }, +[0x2E] = { RM,0, "UCOMISS %x,%X" }, +[0x2F] = { RM,0, "COMISS %x,%X" }, +[0x30] = { 0,0, "WRMSR" }, +[0x31] = { 0,0, "RDTSC" }, +[0x32] = { 0,0, "RDMSR" }, +[0x33] = { 0,0, "RDPMC" }, +[0x42] = { RM,0, "CMOVC %e,%r" }, /* CF */ +[0x43] = { RM,0, "CMOVNC %e,%r" }, /* ¬ CF */ +[0x44] = { RM,0, "CMOVZ %e,%r" }, /* ZF */ +[0x45] = { RM,0, "CMOVNZ %e,%r" }, /* ¬ ZF */ +[0x46] = { RM,0, "CMOVBE %e,%r" }, /* CF ∨ ZF */ +[0x47] = { RM,0, "CMOVA %e,%r" }, /* ¬CF ∧ ¬ZF */ +[0x48] = { RM,0, "CMOVS %e,%r" }, /* SF */ +[0x49] = { RM,0, "CMOVNS %e,%r" }, /* ¬ SF */ +[0x4A] = { RM,0, "CMOVP %e,%r" }, /* PF */ +[0x4B] = { RM,0, "CMOVNP %e,%r" }, /* ¬ PF */ +[0x4C] = { RM,0, "CMOVLT %e,%r" }, /* LT ≡ OF ≠ SF */ +[0x4D] = { RM,0, "CMOVGE %e,%r" }, /* GE ≡ ZF ∨ SF */ +[0x4E] = { RM,0, "CMOVLE %e,%r" }, /* LE ≡ ZF ∨ LT */ +[0x4F] = { RM,0, "CMOVGT %e,%r" }, /* GT ≡ ¬ZF ∧ GE */ +[0x50] = { RM,0, "MOVMSK%s %X,%r" }, /* TO DO: check */ +[0x51] = { RM,0, "SQRT%s %x,%X" }, +[0x52] = { RM,0, "RSQRT%s %x,%X" }, +[0x53] = { RM,0, "RCP%s %x,%X" }, +[0x54] = { RM,0, "AND%s %x,%X" }, +[0x55] = { RM,0, "ANDN%s %x,%X" }, +[0x56] = { RM,0, "OR%s %x,%X" }, /* TO DO: S/D */ +[0x57] = { RM,0, "XOR%s %x,%X" }, /* S/D */ +[0x58] = { RM,0, "ADD%s %x,%X" }, /* S/P S/D */ +[0x59] = { RM,0, "MUL%s %x,%X" }, +[0x5A] = { RM,0, "CVTPS2PD %x,%X" }, +[0x5B] = { RM,0, "CVTPL2PS %x,%X" }, +[0x5C] = { RM,0, "SUB%s %x,%X" }, +[0x5D] = { RM,0, "MIN%s %x,%X" }, +[0x5E] = { RM,0, "DIV%s %x,%X" }, /* TO DO: S/P S/D */ +[0x5F] = { RM,0, "MAX%s %x,%X" }, +[0x60] = { RM,0, "PUNPCKLBW %m,%M" }, +[0x61] = { RM,0, "PUNPCKLWL %m,%M" }, +[0x62] = { RM,0, "PUNPCKLLQ %m,%M" }, +[0x63] = { RM,0, "PACKSSWB %m,%M" }, +[0x64] = { RM,0, "PCMPGTB %m,%M" }, +[0x65] = { RM,0, "PCMPGTW %m,%M" }, +[0x66] = { RM,0, "PCMPGTL %m,%M" }, +[0x67] = { RM,0, "PACKUSWB %m,%M" }, +[0x68] = { RM,0, "PUNPCKHBW %m,%M" }, +[0x69] = { RM,0, "PUNPCKHWL %m,%M" }, +[0x6A] = { RM,0, "PUNPCKHLQ %m,%M" }, +[0x6B] = { RM,0, "PACKSSLW %m,%M" }, +[0x6E] = { RM,0, "MOV%S %e,%M" }, +[0x6F] = { RM,0, "MOVQ %m,%M" }, +[0x70] = { RM,Ib, "PSHUFW %i,%m,%M" }, +[0x74] = { RM,0, "PCMPEQB %m,%M" }, +[0x75] = { RM,0, "PCMPEQW %m,%M" }, +[0x76] = { RM,0, "PCMPEQL %m,%M" }, +[0x7E] = { RM,0, "MOV%S %M,%e" }, +[0x7F] = { RM,0, "MOVQ %M,%m" }, +[0xAE] = { RMOP,0, optab0FAE }, +[0xAA] = { 0,0, "RSM" }, +[0xB0] = { RM,0, "CMPXCHGB %r,%e" }, +[0xB1] = { RM,0, "CMPXCHG%S %r,%e" }, +[0xC0] = { RMB,0, "XADDB %r,%e" }, +[0xC1] = { RM,0, "XADD%S %r,%e" }, +[0xC2] = { RM,Ib, "CMP%s %x,%X,%#i" }, +[0xC3] = { RM,0, "MOVNTI%S %r,%e" }, +[0xC6] = { RM,Ib, "SHUF%s %i,%x,%X" }, +[0xC8] = { 0,0, "BSWAP AX" }, +[0xC9] = { 0,0, "BSWAP CX" }, +[0xCA] = { 0,0, "BSWAP DX" }, +[0xCB] = { 0,0, "BSWAP BX" }, +[0xCC] = { 0,0, "BSWAP SP" }, +[0xCD] = { 0,0, "BSWAP BP" }, +[0xCE] = { 0,0, "BSWAP SI" }, +[0xCF] = { 0,0, "BSWAP DI" }, +[0xD1] = { RM,0, "PSRLW %m,%M" }, +[0xD2] = { RM,0, "PSRLL %m,%M" }, +[0xD3] = { RM,0, "PSRLQ %m,%M" }, +[0xD5] = { RM,0, "PMULLW %m,%M" }, +[0xD6] = { RM,0, "MOVQOZX %m*,%X" }, +[0xD7] = { RM,0, "PMOVMSKB %m,%r" }, +[0xD8] = { RM,0, "PSUBUSB %m,%M" }, +[0xD9] = { RM,0, "PSUBUSW %m,%M" }, +[0xDA] = { RM,0, "PMINUB %m,%M" }, +[0xDB] = { RM,0, "PAND %m,%M" }, +[0xDC] = { RM,0, "PADDUSB %m,%M" }, +[0xDD] = { RM,0, "PADDUSW %m,%M" }, +[0xDE] = { RM,0, "PMAXUB %m,%M" }, +[0xDF] = { RM,0, "PANDN %m,%M" }, +[0xE0] = { RM,0, "PAVGB %m,%M" }, +[0xE1] = { RM,0, "PSRAW %m,%M" }, +[0xE2] = { RM,0, "PSRAL %m,%M" }, +[0xE3] = { RM,0, "PAVGW %m,%M" }, +[0xE4] = { RM,0, "PMULHUW %m,%M" }, +[0xE5] = { RM,0, "PMULHW %m,%M" }, +[0xE7] = { RM,0, "MOVNTQ %M,%e" }, +[0xE8] = { RM,0, "PSUBSB %m,%M" }, +[0xE9] = { RM,0, "PSUBSW %m,%M" }, +[0xEA] = { RM,0, "PMINSW %m,%M" }, +[0xEB] = { RM,0, "POR %m,%M" }, +[0xEC] = { RM,0, "PADDSB %m,%M" }, +[0xED] = { RM,0, "PADDSW %m,%M" }, +[0xEE] = { RM,0, "PMAXSW %m,%M" }, +[0xEF] = { RM,0, "PXOR %m,%M" }, +[0xF1] = { RM,0, "PSLLW %m,%M" }, +[0xF2] = { RM,0, "PSLLL %m,%M" }, +[0xF3] = { RM,0, "PSLLQ %m,%M" }, +[0xF4] = { RM,0, "PMULULQ %m,%M" }, +[0xF5] = { RM,0, "PMADDWL %m,%M" }, +[0xF6] = { RM,0, "PSADBW %m,%M" }, +[0xF7] = { RMR,0, "MASKMOVQ %m,%M" }, +[0xF8] = { RM,0, "PSUBB %m,%M" }, +[0xF9] = { RM,0, "PSUBW %m,%M" }, +[0xFA] = { RM,0, "PSUBL %m,%M" }, +[0xFC] = { RM,0, "PADDB %m,%M" }, +[0xFD] = { RM,0, "PADDW %m,%M" }, +[0xFE] = { RM,0, "PADDL %m,%M" }, + +[0x80] = { Iwds,0, "JOS %p" }, +[0x81] = { Iwds,0, "JOC %p" }, +[0x82] = { Iwds,0, "JCS %p" }, +[0x83] = { Iwds,0, "JCC %p" }, +[0x84] = { Iwds,0, "JEQ %p" }, +[0x85] = { Iwds,0, "JNE %p" }, +[0x86] = { Iwds,0, "JLS %p" }, +[0x87] = { Iwds,0, "JHI %p" }, +[0x88] = { Iwds,0, "JMI %p" }, +[0x89] = { Iwds,0, "JPL %p" }, +[0x8a] = { Iwds,0, "JPS %p" }, +[0x8b] = { Iwds,0, "JPC %p" }, +[0x8c] = { Iwds,0, "JLT %p" }, +[0x8d] = { Iwds,0, "JGE %p" }, +[0x8e] = { Iwds,0, "JLE %p" }, +[0x8f] = { Iwds,0, "JGT %p" }, +[0x90] = { RMB,0, "SETOS %e" }, +[0x91] = { RMB,0, "SETOC %e" }, +[0x92] = { RMB,0, "SETCS %e" }, +[0x93] = { RMB,0, "SETCC %e" }, +[0x94] = { RMB,0, "SETEQ %e" }, +[0x95] = { RMB,0, "SETNE %e" }, +[0x96] = { RMB,0, "SETLS %e" }, +[0x97] = { RMB,0, "SETHI %e" }, +[0x98] = { RMB,0, "SETMI %e" }, +[0x99] = { RMB,0, "SETPL %e" }, +[0x9a] = { RMB,0, "SETPS %e" }, +[0x9b] = { RMB,0, "SETPC %e" }, +[0x9c] = { RMB,0, "SETLT %e" }, +[0x9d] = { RMB,0, "SETGE %e" }, +[0x9e] = { RMB,0, "SETLE %e" }, +[0x9f] = { RMB,0, "SETGT %e" }, +[0xa0] = { 0,0, "PUSHL FS" }, +[0xa1] = { 0,0, "POPL FS" }, +[0xa2] = { 0,0, "CPUID" }, +[0xa3] = { RM,0, "BT%S %r,%e" }, +[0xa4] = { RM,Ib, "SHLD%S %r,%i,%e" }, +[0xa5] = { RM,0, "SHLD%S %r,CL,%e" }, +[0xa8] = { 0,0, "PUSHL GS" }, +[0xa9] = { 0,0, "POPL GS" }, +[0xab] = { RM,0, "BTS%S %r,%e" }, +[0xac] = { RM,Ib, "SHRD%S %r,%i,%e" }, +[0xad] = { RM,0, "SHRD%S %r,CL,%e" }, +[0xaf] = { RM,0, "IMUL%S %e,%r" }, +[0xb2] = { RMM,0, "LSS %e,%r" }, +[0xb3] = { RM,0, "BTR%S %r,%e" }, +[0xb4] = { RMM,0, "LFS %e,%r" }, +[0xb5] = { RMM,0, "LGS %e,%r" }, +[0xb6] = { RMB,0, "MOVBZX %e,%R" }, +[0xb7] = { RM,0, "MOVWZX %e,%R" }, +[0xba] = { RMOP,0, optab0FBA }, +[0xbb] = { RM,0, "BTC%S %e,%r" }, +[0xbc] = { RM,0, "BSF%S %e,%r" }, +[0xbd] = { RM,0, "BSR%S %e,%r" }, +[0xbe] = { RMB,0, "MOVBSX %e,%R" }, +[0xbf] = { RM,0, "MOVWSX %e,%R" }, +[0xc7] = { RMOP,0, optab0FC7 }, }; static Optable optab80[8]= { -[0x00] Ib,0, "ADDB %i,%e", -[0x01] Ib,0, "ORB %i,%e", -[0x02] Ib,0, "ADCB %i,%e", -[0x03] Ib,0, "SBBB %i,%e", -[0x04] Ib,0, "ANDB %i,%e", -[0x05] Ib,0, "SUBB %i,%e", -[0x06] Ib,0, "XORB %i,%e", -[0x07] Ib,0, "CMPB %e,%i", +[0x00] = { Ib,0, "ADDB %i,%e" }, +[0x01] = { Ib,0, "ORB %i,%e" }, +[0x02] = { Ib,0, "ADCB %i,%e" }, +[0x03] = { Ib,0, "SBBB %i,%e" }, +[0x04] = { Ib,0, "ANDB %i,%e" }, +[0x05] = { Ib,0, "SUBB %i,%e" }, +[0x06] = { Ib,0, "XORB %i,%e" }, +[0x07] = { Ib,0, "CMPB %e,%i" }, }; static Optable optab81[8]= { -[0x00] Iwd,0, "ADD%S %i,%e", -[0x01] Iwd,0, "OR%S %i,%e", -[0x02] Iwd,0, "ADC%S %i,%e", -[0x03] Iwd,0, "SBB%S %i,%e", -[0x04] Iwd,0, "AND%S %i,%e", -[0x05] Iwd,0, "SUB%S %i,%e", -[0x06] Iwd,0, "XOR%S %i,%e", -[0x07] Iwd,0, "CMP%S %e,%i", +[0x00] = { Iwd,0, "ADD%S %i,%e" }, +[0x01] = { Iwd,0, "OR%S %i,%e" }, +[0x02] = { Iwd,0, "ADC%S %i,%e" }, +[0x03] = { Iwd,0, "SBB%S %i,%e" }, +[0x04] = { Iwd,0, "AND%S %i,%e" }, +[0x05] = { Iwd,0, "SUB%S %i,%e" }, +[0x06] = { Iwd,0, "XOR%S %i,%e" }, +[0x07] = { Iwd,0, "CMP%S %e,%i" }, }; static Optable optab83[8]= { -[0x00] Ibs,0, "ADD%S %i,%e", -[0x01] Ibs,0, "OR%S %i,%e", -[0x02] Ibs,0, "ADC%S %i,%e", -[0x03] Ibs,0, "SBB%S %i,%e", -[0x04] Ibs,0, "AND%S %i,%e", -[0x05] Ibs,0, "SUB%S %i,%e", -[0x06] Ibs,0, "XOR%S %i,%e", -[0x07] Ibs,0, "CMP%S %e,%i", +[0x00] = { Ibs,0, "ADD%S %i,%e" }, +[0x01] = { Ibs,0, "OR%S %i,%e" }, +[0x02] = { Ibs,0, "ADC%S %i,%e" }, +[0x03] = { Ibs,0, "SBB%S %i,%e" }, +[0x04] = { Ibs,0, "AND%S %i,%e" }, +[0x05] = { Ibs,0, "SUB%S %i,%e" }, +[0x06] = { Ibs,0, "XOR%S %i,%e" }, +[0x07] = { Ibs,0, "CMP%S %e,%i" }, }; static Optable optabC0[8] = { -[0x00] Ib,0, "ROLB %i,%e", -[0x01] Ib,0, "RORB %i,%e", -[0x02] Ib,0, "RCLB %i,%e", -[0x03] Ib,0, "RCRB %i,%e", -[0x04] Ib,0, "SHLB %i,%e", -[0x05] Ib,0, "SHRB %i,%e", -[0x07] Ib,0, "SARB %i,%e", +[0x00] = { Ib,0, "ROLB %i,%e" }, +[0x01] = { Ib,0, "RORB %i,%e" }, +[0x02] = { Ib,0, "RCLB %i,%e" }, +[0x03] = { Ib,0, "RCRB %i,%e" }, +[0x04] = { Ib,0, "SHLB %i,%e" }, +[0x05] = { Ib,0, "SHRB %i,%e" }, +[0x07] = { Ib,0, "SARB %i,%e" }, }; static Optable optabC1[8] = { -[0x00] Ib,0, "ROL%S %i,%e", -[0x01] Ib,0, "ROR%S %i,%e", -[0x02] Ib,0, "RCL%S %i,%e", -[0x03] Ib,0, "RCR%S %i,%e", -[0x04] Ib,0, "SHL%S %i,%e", -[0x05] Ib,0, "SHR%S %i,%e", -[0x07] Ib,0, "SAR%S %i,%e", +[0x00] = { Ib,0, "ROL%S %i,%e" }, +[0x01] = { Ib,0, "ROR%S %i,%e" }, +[0x02] = { Ib,0, "RCL%S %i,%e" }, +[0x03] = { Ib,0, "RCR%S %i,%e" }, +[0x04] = { Ib,0, "SHL%S %i,%e" }, +[0x05] = { Ib,0, "SHR%S %i,%e" }, +[0x07] = { Ib,0, "SAR%S %i,%e" }, }; static Optable optabD0[8] = { -[0x00] 0,0, "ROLB %e", -[0x01] 0,0, "RORB %e", -[0x02] 0,0, "RCLB %e", -[0x03] 0,0, "RCRB %e", -[0x04] 0,0, "SHLB %e", -[0x05] 0,0, "SHRB %e", -[0x07] 0,0, "SARB %e", +[0x00] = { 0,0, "ROLB %e" }, +[0x01] = { 0,0, "RORB %e" }, +[0x02] = { 0,0, "RCLB %e" }, +[0x03] = { 0,0, "RCRB %e" }, +[0x04] = { 0,0, "SHLB %e" }, +[0x05] = { 0,0, "SHRB %e" }, +[0x07] = { 0,0, "SARB %e" }, }; static Optable optabD1[8] = { -[0x00] 0,0, "ROL%S %e", -[0x01] 0,0, "ROR%S %e", -[0x02] 0,0, "RCL%S %e", -[0x03] 0,0, "RCR%S %e", -[0x04] 0,0, "SHL%S %e", -[0x05] 0,0, "SHR%S %e", -[0x07] 0,0, "SAR%S %e", +[0x00] = { 0,0, "ROL%S %e" }, +[0x01] = { 0,0, "ROR%S %e" }, +[0x02] = { 0,0, "RCL%S %e" }, +[0x03] = { 0,0, "RCR%S %e" }, +[0x04] = { 0,0, "SHL%S %e" }, +[0x05] = { 0,0, "SHR%S %e" }, +[0x07] = { 0,0, "SAR%S %e" }, }; static Optable optabD2[8] = { -[0x00] 0,0, "ROLB CL,%e", -[0x01] 0,0, "RORB CL,%e", -[0x02] 0,0, "RCLB CL,%e", -[0x03] 0,0, "RCRB CL,%e", -[0x04] 0,0, "SHLB CL,%e", -[0x05] 0,0, "SHRB CL,%e", -[0x07] 0,0, "SARB CL,%e", +[0x00] = { 0,0, "ROLB CL,%e" }, +[0x01] = { 0,0, "RORB CL,%e" }, +[0x02] = { 0,0, "RCLB CL,%e" }, +[0x03] = { 0,0, "RCRB CL,%e" }, +[0x04] = { 0,0, "SHLB CL,%e" }, +[0x05] = { 0,0, "SHRB CL,%e" }, +[0x07] = { 0,0, "SARB CL,%e" }, }; static Optable optabD3[8] = { -[0x00] 0,0, "ROL%S CL,%e", -[0x01] 0,0, "ROR%S CL,%e", -[0x02] 0,0, "RCL%S CL,%e", -[0x03] 0,0, "RCR%S CL,%e", -[0x04] 0,0, "SHL%S CL,%e", -[0x05] 0,0, "SHR%S CL,%e", -[0x07] 0,0, "SAR%S CL,%e", +[0x00] = { 0,0, "ROL%S CL,%e" }, +[0x01] = { 0,0, "ROR%S CL,%e" }, +[0x02] = { 0,0, "RCL%S CL,%e" }, +[0x03] = { 0,0, "RCR%S CL,%e" }, +[0x04] = { 0,0, "SHL%S CL,%e" }, +[0x05] = { 0,0, "SHR%S CL,%e" }, +[0x07] = { 0,0, "SAR%S CL,%e" }, }; static Optable optabD8[8+8] = { -[0x00] 0,0, "FADDF %e,F0", -[0x01] 0,0, "FMULF %e,F0", -[0x02] 0,0, "FCOMF %e,F0", -[0x03] 0,0, "FCOMFP %e,F0", -[0x04] 0,0, "FSUBF %e,F0", -[0x05] 0,0, "FSUBRF %e,F0", -[0x06] 0,0, "FDIVF %e,F0", -[0x07] 0,0, "FDIVRF %e,F0", -[0x08] 0,0, "FADDD %f,F0", -[0x09] 0,0, "FMULD %f,F0", -[0x0a] 0,0, "FCOMD %f,F0", -[0x0b] 0,0, "FCOMPD %f,F0", -[0x0c] 0,0, "FSUBD %f,F0", -[0x0d] 0,0, "FSUBRD %f,F0", -[0x0e] 0,0, "FDIVD %f,F0", -[0x0f] 0,0, "FDIVRD %f,F0", +[0x00] = { 0,0, "FADDF %e,F0" }, +[0x01] = { 0,0, "FMULF %e,F0" }, +[0x02] = { 0,0, "FCOMF %e,F0" }, +[0x03] = { 0,0, "FCOMFP %e,F0" }, +[0x04] = { 0,0, "FSUBF %e,F0" }, +[0x05] = { 0,0, "FSUBRF %e,F0" }, +[0x06] = { 0,0, "FDIVF %e,F0" }, +[0x07] = { 0,0, "FDIVRF %e,F0" }, +[0x08] = { 0,0, "FADDD %f,F0" }, +[0x09] = { 0,0, "FMULD %f,F0" }, +[0x0a] = { 0,0, "FCOMD %f,F0" }, +[0x0b] = { 0,0, "FCOMPD %f,F0" }, +[0x0c] = { 0,0, "FSUBD %f,F0" }, +[0x0d] = { 0,0, "FSUBRD %f,F0" }, +[0x0e] = { 0,0, "FDIVD %f,F0" }, +[0x0f] = { 0,0, "FDIVRD %f,F0" }, }; /* * optabD9 and optabDB use the following encoding: @@ -940,455 +940,455 @@ static Optable optabD8[8+8] = */ static Optable optabD9[64+8] = { -[0x00] 0,0, "FMOVF %e,F0", -[0x02] 0,0, "FMOVF F0,%e", -[0x03] 0,0, "FMOVFP F0,%e", -[0x04] 0,0, "FLDENV%S %e", -[0x05] 0,0, "FLDCW %e", -[0x06] 0,0, "FSTENV%S %e", -[0x07] 0,0, "FSTCW %e", -[0x08] 0,0, "FMOVD F0,F0", /* Mod R/M = 11xx xxxx*/ -[0x09] 0,0, "FMOVD F1,F0", -[0x0a] 0,0, "FMOVD F2,F0", -[0x0b] 0,0, "FMOVD F3,F0", -[0x0c] 0,0, "FMOVD F4,F0", -[0x0d] 0,0, "FMOVD F5,F0", -[0x0e] 0,0, "FMOVD F6,F0", -[0x0f] 0,0, "FMOVD F7,F0", -[0x10] 0,0, "FXCHD F0,F0", -[0x11] 0,0, "FXCHD F1,F0", -[0x12] 0,0, "FXCHD F2,F0", -[0x13] 0,0, "FXCHD F3,F0", -[0x14] 0,0, "FXCHD F4,F0", -[0x15] 0,0, "FXCHD F5,F0", -[0x16] 0,0, "FXCHD F6,F0", -[0x17] 0,0, "FXCHD F7,F0", -[0x18] 0,0, "FNOP", -[0x28] 0,0, "FCHS", -[0x29] 0,0, "FABS", -[0x2c] 0,0, "FTST", -[0x2d] 0,0, "FXAM", -[0x30] 0,0, "FLD1", -[0x31] 0,0, "FLDL2T", -[0x32] 0,0, "FLDL2E", -[0x33] 0,0, "FLDPI", -[0x34] 0,0, "FLDLG2", -[0x35] 0,0, "FLDLN2", -[0x36] 0,0, "FLDZ", -[0x38] 0,0, "F2XM1", -[0x39] 0,0, "FYL2X", -[0x3a] 0,0, "FPTAN", -[0x3b] 0,0, "FPATAN", -[0x3c] 0,0, "FXTRACT", -[0x3d] 0,0, "FPREM1", -[0x3e] 0,0, "FDECSTP", -[0x3f] 0,0, "FNCSTP", -[0x40] 0,0, "FPREM", -[0x41] 0,0, "FYL2XP1", -[0x42] 0,0, "FSQRT", -[0x43] 0,0, "FSINCOS", -[0x44] 0,0, "FRNDINT", -[0x45] 0,0, "FSCALE", -[0x46] 0,0, "FSIN", -[0x47] 0,0, "FCOS", +[0x00] = { 0,0, "FMOVF %e,F0" }, +[0x02] = { 0,0, "FMOVF F0,%e" }, +[0x03] = { 0,0, "FMOVFP F0,%e" }, +[0x04] = { 0,0, "FLDENV%S %e" }, +[0x05] = { 0,0, "FLDCW %e" }, +[0x06] = { 0,0, "FSTENV%S %e" }, +[0x07] = { 0,0, "FSTCW %e" }, +[0x08] = { 0,0, "FMOVD F0,F0" }, /* Mod R/M = 11xx xxxx*/ +[0x09] = { 0,0, "FMOVD F1,F0" }, +[0x0a] = { 0,0, "FMOVD F2,F0" }, +[0x0b] = { 0,0, "FMOVD F3,F0" }, +[0x0c] = { 0,0, "FMOVD F4,F0" }, +[0x0d] = { 0,0, "FMOVD F5,F0" }, +[0x0e] = { 0,0, "FMOVD F6,F0" }, +[0x0f] = { 0,0, "FMOVD F7,F0" }, +[0x10] = { 0,0, "FXCHD F0,F0" }, +[0x11] = { 0,0, "FXCHD F1,F0" }, +[0x12] = { 0,0, "FXCHD F2,F0" }, +[0x13] = { 0,0, "FXCHD F3,F0" }, +[0x14] = { 0,0, "FXCHD F4,F0" }, +[0x15] = { 0,0, "FXCHD F5,F0" }, +[0x16] = { 0,0, "FXCHD F6,F0" }, +[0x17] = { 0,0, "FXCHD F7,F0" }, +[0x18] = { 0,0, "FNOP" }, +[0x28] = { 0,0, "FCHS" }, +[0x29] = { 0,0, "FABS" }, +[0x2c] = { 0,0, "FTST" }, +[0x2d] = { 0,0, "FXAM" }, +[0x30] = { 0,0, "FLD1" }, +[0x31] = { 0,0, "FLDL2T" }, +[0x32] = { 0,0, "FLDL2E" }, +[0x33] = { 0,0, "FLDPI" }, +[0x34] = { 0,0, "FLDLG2" }, +[0x35] = { 0,0, "FLDLN2" }, +[0x36] = { 0,0, "FLDZ" }, +[0x38] = { 0,0, "F2XM1" }, +[0x39] = { 0,0, "FYL2X" }, +[0x3a] = { 0,0, "FPTAN" }, +[0x3b] = { 0,0, "FPATAN" }, +[0x3c] = { 0,0, "FXTRACT" }, +[0x3d] = { 0,0, "FPREM1" }, +[0x3e] = { 0,0, "FDECSTP" }, +[0x3f] = { 0,0, "FNCSTP" }, +[0x40] = { 0,0, "FPREM" }, +[0x41] = { 0,0, "FYL2XP1" }, +[0x42] = { 0,0, "FSQRT" }, +[0x43] = { 0,0, "FSINCOS" }, +[0x44] = { 0,0, "FRNDINT" }, +[0x45] = { 0,0, "FSCALE" }, +[0x46] = { 0,0, "FSIN" }, +[0x47] = { 0,0, "FCOS" }, }; static Optable optabDA[8+8] = { -[0x00] 0,0, "FADDL %e,F0", -[0x01] 0,0, "FMULL %e,F0", -[0x02] 0,0, "FCOML %e,F0", -[0x03] 0,0, "FCOMLP %e,F0", -[0x04] 0,0, "FSUBL %e,F0", -[0x05] 0,0, "FSUBRL %e,F0", -[0x06] 0,0, "FDIVL %e,F0", -[0x07] 0,0, "FDIVRL %e,F0", -[0x08] 0,0, "FCMOVCS %f,F0", -[0x09] 0,0, "FCMOVEQ %f,F0", -[0x0a] 0,0, "FCMOVLS %f,F0", -[0x0b] 0,0, "FCMOVUN %f,F0", -[0x0d] Op_R1,0, "FUCOMPP", +[0x00] = { 0,0, "FADDL %e,F0" }, +[0x01] = { 0,0, "FMULL %e,F0" }, +[0x02] = { 0,0, "FCOML %e,F0" }, +[0x03] = { 0,0, "FCOMLP %e,F0" }, +[0x04] = { 0,0, "FSUBL %e,F0" }, +[0x05] = { 0,0, "FSUBRL %e,F0" }, +[0x06] = { 0,0, "FDIVL %e,F0" }, +[0x07] = { 0,0, "FDIVRL %e,F0" }, +[0x08] = { 0,0, "FCMOVCS %f,F0" }, +[0x09] = { 0,0, "FCMOVEQ %f,F0" }, +[0x0a] = { 0,0, "FCMOVLS %f,F0" }, +[0x0b] = { 0,0, "FCMOVUN %f,F0" }, +[0x0d] = { Op_R1,0, "FUCOMPP" }, }; static Optable optabDB[8+64] = { -[0x00] 0,0, "FMOVL %e,F0", -[0x02] 0,0, "FMOVL F0,%e", -[0x03] 0,0, "FMOVLP F0,%e", -[0x05] 0,0, "FMOVX %e,F0", -[0x07] 0,0, "FMOVXP F0,%e", -[0x08] 0,0, "FCMOVCC %f,F0", -[0x09] 0,0, "FCMOVNE %f,F0", -[0x0a] 0,0, "FCMOVHI %f,F0", -[0x0b] 0,0, "FCMOVNU %f,F0", -[0x0d] 0,0, "FUCOMI F0,%f", -[0x0e] 0,0, "FCOMI F0,%f", -[0x2a] 0,0, "FCLEX", -[0x2b] 0,0, "FINIT", +[0x00] = { 0,0, "FMOVL %e,F0" }, +[0x02] = { 0,0, "FMOVL F0,%e" }, +[0x03] = { 0,0, "FMOVLP F0,%e" }, +[0x05] = { 0,0, "FMOVX %e,F0" }, +[0x07] = { 0,0, "FMOVXP F0,%e" }, +[0x08] = { 0,0, "FCMOVCC %f,F0" }, +[0x09] = { 0,0, "FCMOVNE %f,F0" }, +[0x0a] = { 0,0, "FCMOVHI %f,F0" }, +[0x0b] = { 0,0, "FCMOVNU %f,F0" }, +[0x0d] = { 0,0, "FUCOMI F0,%f" }, +[0x0e] = { 0,0, "FCOMI F0,%f" }, +[0x2a] = { 0,0, "FCLEX" }, +[0x2b] = { 0,0, "FINIT" }, }; static Optable optabDC[8+8] = { -[0x00] 0,0, "FADDD %e,F0", -[0x01] 0,0, "FMULD %e,F0", -[0x02] 0,0, "FCOMD %e,F0", -[0x03] 0,0, "FCOMDP %e,F0", -[0x04] 0,0, "FSUBD %e,F0", -[0x05] 0,0, "FSUBRD %e,F0", -[0x06] 0,0, "FDIVD %e,F0", -[0x07] 0,0, "FDIVRD %e,F0", -[0x08] 0,0, "FADDD F0,%f", -[0x09] 0,0, "FMULD F0,%f", -[0x0c] 0,0, "FSUBRD F0,%f", -[0x0d] 0,0, "FSUBD F0,%f", -[0x0e] 0,0, "FDIVRD F0,%f", -[0x0f] 0,0, "FDIVD F0,%f", +[0x00] = { 0,0, "FADDD %e,F0" }, +[0x01] = { 0,0, "FMULD %e,F0" }, +[0x02] = { 0,0, "FCOMD %e,F0" }, +[0x03] = { 0,0, "FCOMDP %e,F0" }, +[0x04] = { 0,0, "FSUBD %e,F0" }, +[0x05] = { 0,0, "FSUBRD %e,F0" }, +[0x06] = { 0,0, "FDIVD %e,F0" }, +[0x07] = { 0,0, "FDIVRD %e,F0" }, +[0x08] = { 0,0, "FADDD F0,%f" }, +[0x09] = { 0,0, "FMULD F0,%f" }, +[0x0c] = { 0,0, "FSUBRD F0,%f" }, +[0x0d] = { 0,0, "FSUBD F0,%f" }, +[0x0e] = { 0,0, "FDIVRD F0,%f" }, +[0x0f] = { 0,0, "FDIVD F0,%f" }, }; static Optable optabDD[8+8] = { -[0x00] 0,0, "FMOVD %e,F0", -[0x02] 0,0, "FMOVD F0,%e", -[0x03] 0,0, "FMOVDP F0,%e", -[0x04] 0,0, "FRSTOR%S %e", -[0x06] 0,0, "FSAVE%S %e", -[0x07] 0,0, "FSTSW %e", -[0x08] 0,0, "FFREED %f", -[0x0a] 0,0, "FMOVD %f,F0", -[0x0b] 0,0, "FMOVDP %f,F0", -[0x0c] 0,0, "FUCOMD %f,F0", -[0x0d] 0,0, "FUCOMDP %f,F0", +[0x00] = { 0,0, "FMOVD %e,F0" }, +[0x02] = { 0,0, "FMOVD F0,%e" }, +[0x03] = { 0,0, "FMOVDP F0,%e" }, +[0x04] = { 0,0, "FRSTOR%S %e" }, +[0x06] = { 0,0, "FSAVE%S %e" }, +[0x07] = { 0,0, "FSTSW %e" }, +[0x08] = { 0,0, "FFREED %f" }, +[0x0a] = { 0,0, "FMOVD %f,F0" }, +[0x0b] = { 0,0, "FMOVDP %f,F0" }, +[0x0c] = { 0,0, "FUCOMD %f,F0" }, +[0x0d] = { 0,0, "FUCOMDP %f,F0" }, }; static Optable optabDE[8+8] = { -[0x00] 0,0, "FADDW %e,F0", -[0x01] 0,0, "FMULW %e,F0", -[0x02] 0,0, "FCOMW %e,F0", -[0x03] 0,0, "FCOMWP %e,F0", -[0x04] 0,0, "FSUBW %e,F0", -[0x05] 0,0, "FSUBRW %e,F0", -[0x06] 0,0, "FDIVW %e,F0", -[0x07] 0,0, "FDIVRW %e,F0", -[0x08] 0,0, "FADDDP F0,%f", -[0x09] 0,0, "FMULDP F0,%f", -[0x0b] Op_R1,0, "FCOMPDP", -[0x0c] 0,0, "FSUBRDP F0,%f", -[0x0d] 0,0, "FSUBDP F0,%f", -[0x0e] 0,0, "FDIVRDP F0,%f", -[0x0f] 0,0, "FDIVDP F0,%f", +[0x00] = { 0,0, "FADDW %e,F0" }, +[0x01] = { 0,0, "FMULW %e,F0" }, +[0x02] = { 0,0, "FCOMW %e,F0" }, +[0x03] = { 0,0, "FCOMWP %e,F0" }, +[0x04] = { 0,0, "FSUBW %e,F0" }, +[0x05] = { 0,0, "FSUBRW %e,F0" }, +[0x06] = { 0,0, "FDIVW %e,F0" }, +[0x07] = { 0,0, "FDIVRW %e,F0" }, +[0x08] = { 0,0, "FADDDP F0,%f" }, +[0x09] = { 0,0, "FMULDP F0,%f" }, +[0x0b] = { Op_R1,0, "FCOMPDP" }, +[0x0c] = { 0,0, "FSUBRDP F0,%f" }, +[0x0d] = { 0,0, "FSUBDP F0,%f" }, +[0x0e] = { 0,0, "FDIVRDP F0,%f" }, +[0x0f] = { 0,0, "FDIVDP F0,%f" }, }; static Optable optabDF[8+8] = { -[0x00] 0,0, "FMOVW %e,F0", -[0x02] 0,0, "FMOVW F0,%e", -[0x03] 0,0, "FMOVWP F0,%e", -[0x04] 0,0, "FBLD %e", -[0x05] 0,0, "FMOVL %e,F0", -[0x06] 0,0, "FBSTP %e", -[0x07] 0,0, "FMOVLP F0,%e", -[0x0c] Op_R0,0, "FSTSW %OAX", -[0x0d] 0,0, "FUCOMIP F0,%f", -[0x0e] 0,0, "FCOMIP F0,%f", +[0x00] = { 0,0, "FMOVW %e,F0" }, +[0x02] = { 0,0, "FMOVW F0,%e" }, +[0x03] = { 0,0, "FMOVWP F0,%e" }, +[0x04] = { 0,0, "FBLD %e" }, +[0x05] = { 0,0, "FMOVL %e,F0" }, +[0x06] = { 0,0, "FBSTP %e" }, +[0x07] = { 0,0, "FMOVLP F0,%e" }, +[0x0c] = { Op_R0,0, "FSTSW %OAX" }, +[0x0d] = { 0,0, "FUCOMIP F0,%f" }, +[0x0e] = { 0,0, "FCOMIP F0,%f" }, }; static Optable optabF6[8] = { -[0x00] Ib,0, "TESTB %i,%e", -[0x02] 0,0, "NOTB %e", -[0x03] 0,0, "NEGB %e", -[0x04] 0,0, "MULB AL,%e", -[0x05] 0,0, "IMULB AL,%e", -[0x06] 0,0, "DIVB AL,%e", -[0x07] 0,0, "IDIVB AL,%e", +[0x00] = { Ib,0, "TESTB %i,%e" }, +[0x02] = { 0,0, "NOTB %e" }, +[0x03] = { 0,0, "NEGB %e" }, +[0x04] = { 0,0, "MULB AL,%e" }, +[0x05] = { 0,0, "IMULB AL,%e" }, +[0x06] = { 0,0, "DIVB AL,%e" }, +[0x07] = { 0,0, "IDIVB AL,%e" }, }; static Optable optabF7[8] = { -[0x00] Iwd,0, "TEST%S %i,%e", -[0x02] 0,0, "NOT%S %e", -[0x03] 0,0, "NEG%S %e", -[0x04] 0,0, "MUL%S %OAX,%e", -[0x05] 0,0, "IMUL%S %OAX,%e", -[0x06] 0,0, "DIV%S %OAX,%e", -[0x07] 0,0, "IDIV%S %OAX,%e", +[0x00] = { Iwd,0, "TEST%S %i,%e" }, +[0x02] = { 0,0, "NOT%S %e" }, +[0x03] = { 0,0, "NEG%S %e" }, +[0x04] = { 0,0, "MUL%S %OAX,%e" }, +[0x05] = { 0,0, "IMUL%S %OAX,%e" }, +[0x06] = { 0,0, "DIV%S %OAX,%e" }, +[0x07] = { 0,0, "IDIV%S %OAX,%e" }, }; static Optable optabFE[8] = { -[0x00] 0,0, "INCB %e", -[0x01] 0,0, "DECB %e", +[0x00] = { 0,0, "INCB %e" }, +[0x01] = { 0,0, "DECB %e" }, }; static Optable optabFF[8] = { -[0x00] 0,0, "INC%S %e", -[0x01] 0,0, "DEC%S %e", -[0x02] JUMP,0, "CALL* %e", -[0x03] JUMP,0, "CALLF* %e", -[0x04] JUMP,0, "JMP* %e", -[0x05] JUMP,0, "JMPF* %e", -[0x06] 0,0, "PUSHL %e", +[0x00] = { 0,0, "INC%S %e" }, +[0x01] = { 0,0, "DEC%S %e" }, +[0x02] = { JUMP,0, "CALL* %e" }, +[0x03] = { JUMP,0, "CALLF* %e" }, +[0x04] = { JUMP,0, "JMP* %e" }, +[0x05] = { JUMP,0, "JMPF* %e" }, +[0x06] = { 0,0, "PUSHL %e" }, }; static Optable optable[256+2] = { -[0x00] RMB,0, "ADDB %r,%e", -[0x01] RM,0, "ADD%S %r,%e", -[0x02] RMB,0, "ADDB %e,%r", -[0x03] RM,0, "ADD%S %e,%r", -[0x04] Ib,0, "ADDB %i,AL", -[0x05] Iwd,0, "ADD%S %i,%OAX", -[0x06] 0,0, "PUSHL ES", -[0x07] 0,0, "POPL ES", -[0x08] RMB,0, "ORB %r,%e", -[0x09] RM,0, "OR%S %r,%e", -[0x0a] RMB,0, "ORB %e,%r", -[0x0b] RM,0, "OR%S %e,%r", -[0x0c] Ib,0, "ORB %i,AL", -[0x0d] Iwd,0, "OR%S %i,%OAX", -[0x0e] 0,0, "PUSHL CS", -[0x0f] AUXMM,0, optab0F, -[0x10] RMB,0, "ADCB %r,%e", -[0x11] RM,0, "ADC%S %r,%e", -[0x12] RMB,0, "ADCB %e,%r", -[0x13] RM,0, "ADC%S %e,%r", -[0x14] Ib,0, "ADCB %i,AL", -[0x15] Iwd,0, "ADC%S %i,%OAX", -[0x16] 0,0, "PUSHL SS", -[0x17] 0,0, "POPL SS", -[0x18] RMB,0, "SBBB %r,%e", -[0x19] RM,0, "SBB%S %r,%e", -[0x1a] RMB,0, "SBBB %e,%r", -[0x1b] RM,0, "SBB%S %e,%r", -[0x1c] Ib,0, "SBBB %i,AL", -[0x1d] Iwd,0, "SBB%S %i,%OAX", -[0x1e] 0,0, "PUSHL DS", -[0x1f] 0,0, "POPL DS", -[0x20] RMB,0, "ANDB %r,%e", -[0x21] RM,0, "AND%S %r,%e", -[0x22] RMB,0, "ANDB %e,%r", -[0x23] RM,0, "AND%S %e,%r", -[0x24] Ib,0, "ANDB %i,AL", -[0x25] Iwd,0, "AND%S %i,%OAX", -[0x26] SEG,0, "ES:", -[0x27] 0,0, "DAA", -[0x28] RMB,0, "SUBB %r,%e", -[0x29] RM,0, "SUB%S %r,%e", -[0x2a] RMB,0, "SUBB %e,%r", -[0x2b] RM,0, "SUB%S %e,%r", -[0x2c] Ib,0, "SUBB %i,AL", -[0x2d] Iwd,0, "SUB%S %i,%OAX", -[0x2e] SEG,0, "CS:", -[0x2f] 0,0, "DAS", -[0x30] RMB,0, "XORB %r,%e", -[0x31] RM,0, "XOR%S %r,%e", -[0x32] RMB,0, "XORB %e,%r", -[0x33] RM,0, "XOR%S %e,%r", -[0x34] Ib,0, "XORB %i,AL", -[0x35] Iwd,0, "XOR%S %i,%OAX", -[0x36] SEG,0, "SS:", -[0x37] 0,0, "AAA", -[0x38] RMB,0, "CMPB %r,%e", -[0x39] RM,0, "CMP%S %r,%e", -[0x3a] RMB,0, "CMPB %e,%r", -[0x3b] RM,0, "CMP%S %e,%r", -[0x3c] Ib,0, "CMPB %i,AL", -[0x3d] Iwd,0, "CMP%S %i,%OAX", -[0x3e] SEG,0, "DS:", -[0x3f] 0,0, "AAS", -[0x40] 0,0, "INC%S %OAX", -[0x41] 0,0, "INC%S %OCX", -[0x42] 0,0, "INC%S %ODX", -[0x43] 0,0, "INC%S %OBX", -[0x44] 0,0, "INC%S %OSP", -[0x45] 0,0, "INC%S %OBP", -[0x46] 0,0, "INC%S %OSI", -[0x47] 0,0, "INC%S %ODI", -[0x48] 0,0, "DEC%S %OAX", -[0x49] 0,0, "DEC%S %OCX", -[0x4a] 0,0, "DEC%S %ODX", -[0x4b] 0,0, "DEC%S %OBX", -[0x4c] 0,0, "DEC%S %OSP", -[0x4d] 0,0, "DEC%S %OBP", -[0x4e] 0,0, "DEC%S %OSI", -[0x4f] 0,0, "DEC%S %ODI", -[0x50] 0,0, "PUSH%S %OAX", -[0x51] 0,0, "PUSH%S %OCX", -[0x52] 0,0, "PUSH%S %ODX", -[0x53] 0,0, "PUSH%S %OBX", -[0x54] 0,0, "PUSH%S %OSP", -[0x55] 0,0, "PUSH%S %OBP", -[0x56] 0,0, "PUSH%S %OSI", -[0x57] 0,0, "PUSH%S %ODI", -[0x58] 0,0, "POP%S %OAX", -[0x59] 0,0, "POP%S %OCX", -[0x5a] 0,0, "POP%S %ODX", -[0x5b] 0,0, "POP%S %OBX", -[0x5c] 0,0, "POP%S %OSP", -[0x5d] 0,0, "POP%S %OBP", -[0x5e] 0,0, "POP%S %OSI", -[0x5f] 0,0, "POP%S %ODI", -[0x60] 0,0, "PUSHA%S", -[0x61] 0,0, "POPA%S", -[0x62] RMM,0, "BOUND %e,%r", -[0x63] RM,0, "ARPL %r,%e", -[0x64] SEG,0, "FS:", -[0x65] SEG,0, "GS:", -[0x66] OPOVER,0, "", -[0x67] ADDOVER,0, "", -[0x68] Iwd,0, "PUSH%S %i", -[0x69] RM,Iwd, "IMUL%S %e,%i,%r", -[0x6a] Ib,0, "PUSH%S %i", -[0x6b] RM,Ibs, "IMUL%S %e,%i,%r", -[0x6c] 0,0, "INSB DX,(%ODI)", -[0x6d] 0,0, "INS%S DX,(%ODI)", -[0x6e] 0,0, "OUTSB (%ASI),DX", -[0x6f] 0,0, "OUTS%S (%ASI),DX", -[0x70] Jbs,0, "JOS %p", -[0x71] Jbs,0, "JOC %p", -[0x72] Jbs,0, "JCS %p", -[0x73] Jbs,0, "JCC %p", -[0x74] Jbs,0, "JEQ %p", -[0x75] Jbs,0, "JNE %p", -[0x76] Jbs,0, "JLS %p", -[0x77] Jbs,0, "JHI %p", -[0x78] Jbs,0, "JMI %p", -[0x79] Jbs,0, "JPL %p", -[0x7a] Jbs,0, "JPS %p", -[0x7b] Jbs,0, "JPC %p", -[0x7c] Jbs,0, "JLT %p", -[0x7d] Jbs,0, "JGE %p", -[0x7e] Jbs,0, "JLE %p", -[0x7f] Jbs,0, "JGT %p", -[0x80] RMOPB,0, optab80, -[0x81] RMOP,0, optab81, -[0x83] RMOP,0, optab83, -[0x84] RMB,0, "TESTB %r,%e", -[0x85] RM,0, "TEST%S %r,%e", -[0x86] RMB,0, "XCHGB %r,%e", -[0x87] RM,0, "XCHG%S %r,%e", -[0x88] RMB,0, "MOVB %r,%e", -[0x89] RM,0, "MOV%S %r,%e", -[0x8a] RMB,0, "MOVB %e,%r", -[0x8b] RM,0, "MOV%S %e,%r", -[0x8c] RM,0, "MOVW %g,%e", -[0x8d] RM,0, "LEA%S %e,%r", -[0x8e] RM,0, "MOVW %e,%g", -[0x8f] RM,0, "POP%S %e", -[0x90] 0,0, "NOP", -[0x91] 0,0, "XCHG %OCX,%OAX", -[0x92] 0,0, "XCHG %ODX,%OAX", -[0x93] 0,0, "XCHG %OBX,%OAX", -[0x94] 0,0, "XCHG %OSP,%OAX", -[0x95] 0,0, "XCHG %OBP,%OAX", -[0x96] 0,0, "XCHG %OSI,%OAX", -[0x97] 0,0, "XCHG %ODI,%OAX", -[0x98] 0,0, "%W", /* miserable CBW or CWDE */ -[0x99] 0,0, "%w", /* idiotic CWD or CDQ */ -[0x9a] PTR,0, "CALL%S %d", -[0x9b] 0,0, "WAIT", -[0x9c] 0,0, "PUSHF", -[0x9d] 0,0, "POPF", -[0x9e] 0,0, "SAHF", -[0x9f] 0,0, "LAHF", -[0xa0] Awd,0, "MOVB %i,AL", -[0xa1] Awd,0, "MOV%S %i,%OAX", -[0xa2] Awd,0, "MOVB AL,%i", -[0xa3] Awd,0, "MOV%S %OAX,%i", -[0xa4] 0,0, "MOVSB (%ASI),(%ADI)", -[0xa5] 0,0, "MOVS%S (%ASI),(%ADI)", -[0xa6] 0,0, "CMPSB (%ASI),(%ADI)", -[0xa7] 0,0, "CMPS%S (%ASI),(%ADI)", -[0xa8] Ib,0, "TESTB %i,AL", -[0xa9] Iwd,0, "TEST%S %i,%OAX", -[0xaa] 0,0, "STOSB AL,(%ADI)", -[0xab] 0,0, "STOS%S %OAX,(%ADI)", -[0xac] 0,0, "LODSB (%ASI),AL", -[0xad] 0,0, "LODS%S (%ASI),%OAX", -[0xae] 0,0, "SCASB (%ADI),AL", -[0xaf] 0,0, "SCAS%S (%ADI),%OAX", -[0xb0] Ib,0, "MOVB %i,AL", -[0xb1] Ib,0, "MOVB %i,CL", -[0xb2] Ib,0, "MOVB %i,DL", -[0xb3] Ib,0, "MOVB %i,BL", -[0xb4] Ib,0, "MOVB %i,AH", -[0xb5] Ib,0, "MOVB %i,CH", -[0xb6] Ib,0, "MOVB %i,DH", -[0xb7] Ib,0, "MOVB %i,BH", -[0xb8] Iwdq,0, "MOV%S %i,%OAX", -[0xb9] Iwdq,0, "MOV%S %i,%OCX", -[0xba] Iwdq,0, "MOV%S %i,%ODX", -[0xbb] Iwdq,0, "MOV%S %i,%OBX", -[0xbc] Iwdq,0, "MOV%S %i,%OSP", -[0xbd] Iwdq,0, "MOV%S %i,%OBP", -[0xbe] Iwdq,0, "MOV%S %i,%OSI", -[0xbf] Iwdq,0, "MOV%S %i,%ODI", -[0xc0] RMOPB,0, optabC0, -[0xc1] RMOP,0, optabC1, -[0xc2] Iw,0, "RET %i", -[0xc3] RET,0, "RET", -[0xc4] RM,0, "LES %e,%r", -[0xc5] RM,0, "LDS %e,%r", -[0xc6] RMB,Ib, "MOVB %i,%e", -[0xc7] RM,Iwd, "MOV%S %i,%e", -[0xc8] Iw2,Ib, "ENTER %i,%I", /* loony ENTER */ -[0xc9] RET,0, "LEAVE", /* bizarre LEAVE */ -[0xca] Iw,0, "RETF %i", -[0xcb] RET,0, "RETF", -[0xcc] 0,0, "INT 3", -[0xcd] Ib,0, "INTB %i", -[0xce] 0,0, "INTO", -[0xcf] 0,0, "IRET", -[0xd0] RMOPB,0, optabD0, -[0xd1] RMOP,0, optabD1, -[0xd2] RMOPB,0, optabD2, -[0xd3] RMOP,0, optabD3, -[0xd4] OA,0, "AAM", -[0xd5] OA,0, "AAD", -[0xd7] 0,0, "XLAT", -[0xd8] FRMOP,0, optabD8, -[0xd9] FRMEX,0, optabD9, -[0xda] FRMOP,0, optabDA, -[0xdb] FRMEX,0, optabDB, -[0xdc] FRMOP,0, optabDC, -[0xdd] FRMOP,0, optabDD, -[0xde] FRMOP,0, optabDE, -[0xdf] FRMOP,0, optabDF, -[0xe0] Jbs,0, "LOOPNE %p", -[0xe1] Jbs,0, "LOOPE %p", -[0xe2] Jbs,0, "LOOP %p", -[0xe3] Jbs,0, "JCXZ %p", -[0xe4] Ib,0, "INB %i,AL", -[0xe5] Ib,0, "IN%S %i,%OAX", -[0xe6] Ib,0, "OUTB AL,%i", -[0xe7] Ib,0, "OUT%S %OAX,%i", -[0xe8] Iwds,0, "CALL %p", -[0xe9] Iwds,0, "JMP %p", -[0xea] PTR,0, "JMP %d", -[0xeb] Jbs,0, "JMP %p", -[0xec] 0,0, "INB DX,AL", -[0xed] 0,0, "IN%S DX,%OAX", -[0xee] 0,0, "OUTB AL,DX", -[0xef] 0,0, "OUT%S %OAX,DX", -[0xf0] PRE,0, "LOCK", -[0xf2] OPRE,0, "REPNE", -[0xf3] OPRE,0, "REP", -[0xf4] 0,0, "HLT", -[0xf5] 0,0, "CMC", -[0xf6] RMOPB,0, optabF6, -[0xf7] RMOP,0, optabF7, -[0xf8] 0,0, "CLC", -[0xf9] 0,0, "STC", -[0xfa] 0,0, "CLI", -[0xfb] 0,0, "STI", -[0xfc] 0,0, "CLD", -[0xfd] 0,0, "STD", -[0xfe] RMOPB,0, optabFE, -[0xff] RMOP,0, optabFF, -[0x100] RM,0, "MOVLQSX %e,%r", -[0x101] RM,0, "MOVLQZX %e,%r", +[0x00] = { RMB,0, "ADDB %r,%e" }, +[0x01] = { RM,0, "ADD%S %r,%e" }, +[0x02] = { RMB,0, "ADDB %e,%r" }, +[0x03] = { RM,0, "ADD%S %e,%r" }, +[0x04] = { Ib,0, "ADDB %i,AL" }, +[0x05] = { Iwd,0, "ADD%S %i,%OAX" }, +[0x06] = { 0,0, "PUSHL ES" }, +[0x07] = { 0,0, "POPL ES" }, +[0x08] = { RMB,0, "ORB %r,%e" }, +[0x09] = { RM,0, "OR%S %r,%e" }, +[0x0a] = { RMB,0, "ORB %e,%r" }, +[0x0b] = { RM,0, "OR%S %e,%r" }, +[0x0c] = { Ib,0, "ORB %i,AL" }, +[0x0d] = { Iwd,0, "OR%S %i,%OAX" }, +[0x0e] = { 0,0, "PUSHL CS" }, +[0x0f] = { AUXMM,0, optab0F }, +[0x10] = { RMB,0, "ADCB %r,%e" }, +[0x11] = { RM,0, "ADC%S %r,%e" }, +[0x12] = { RMB,0, "ADCB %e,%r" }, +[0x13] = { RM,0, "ADC%S %e,%r" }, +[0x14] = { Ib,0, "ADCB %i,AL" }, +[0x15] = { Iwd,0, "ADC%S %i,%OAX" }, +[0x16] = { 0,0, "PUSHL SS" }, +[0x17] = { 0,0, "POPL SS" }, +[0x18] = { RMB,0, "SBBB %r,%e" }, +[0x19] = { RM,0, "SBB%S %r,%e" }, +[0x1a] = { RMB,0, "SBBB %e,%r" }, +[0x1b] = { RM,0, "SBB%S %e,%r" }, +[0x1c] = { Ib,0, "SBBB %i,AL" }, +[0x1d] = { Iwd,0, "SBB%S %i,%OAX" }, +[0x1e] = { 0,0, "PUSHL DS" }, +[0x1f] = { 0,0, "POPL DS" }, +[0x20] = { RMB,0, "ANDB %r,%e" }, +[0x21] = { RM,0, "AND%S %r,%e" }, +[0x22] = { RMB,0, "ANDB %e,%r" }, +[0x23] = { RM,0, "AND%S %e,%r" }, +[0x24] = { Ib,0, "ANDB %i,AL" }, +[0x25] = { Iwd,0, "AND%S %i,%OAX" }, +[0x26] = { SEG,0, "ES:" }, +[0x27] = { 0,0, "DAA" }, +[0x28] = { RMB,0, "SUBB %r,%e" }, +[0x29] = { RM,0, "SUB%S %r,%e" }, +[0x2a] = { RMB,0, "SUBB %e,%r" }, +[0x2b] = { RM,0, "SUB%S %e,%r" }, +[0x2c] = { Ib,0, "SUBB %i,AL" }, +[0x2d] = { Iwd,0, "SUB%S %i,%OAX" }, +[0x2e] = { SEG,0, "CS:" }, +[0x2f] = { 0,0, "DAS" }, +[0x30] = { RMB,0, "XORB %r,%e" }, +[0x31] = { RM,0, "XOR%S %r,%e" }, +[0x32] = { RMB,0, "XORB %e,%r" }, +[0x33] = { RM,0, "XOR%S %e,%r" }, +[0x34] = { Ib,0, "XORB %i,AL" }, +[0x35] = { Iwd,0, "XOR%S %i,%OAX" }, +[0x36] = { SEG,0, "SS:" }, +[0x37] = { 0,0, "AAA" }, +[0x38] = { RMB,0, "CMPB %r,%e" }, +[0x39] = { RM,0, "CMP%S %r,%e" }, +[0x3a] = { RMB,0, "CMPB %e,%r" }, +[0x3b] = { RM,0, "CMP%S %e,%r" }, +[0x3c] = { Ib,0, "CMPB %i,AL" }, +[0x3d] = { Iwd,0, "CMP%S %i,%OAX" }, +[0x3e] = { SEG,0, "DS:" }, +[0x3f] = { 0,0, "AAS" }, +[0x40] = { 0,0, "INC%S %OAX" }, +[0x41] = { 0,0, "INC%S %OCX" }, +[0x42] = { 0,0, "INC%S %ODX" }, +[0x43] = { 0,0, "INC%S %OBX" }, +[0x44] = { 0,0, "INC%S %OSP" }, +[0x45] = { 0,0, "INC%S %OBP" }, +[0x46] = { 0,0, "INC%S %OSI" }, +[0x47] = { 0,0, "INC%S %ODI" }, +[0x48] = { 0,0, "DEC%S %OAX" }, +[0x49] = { 0,0, "DEC%S %OCX" }, +[0x4a] = { 0,0, "DEC%S %ODX" }, +[0x4b] = { 0,0, "DEC%S %OBX" }, +[0x4c] = { 0,0, "DEC%S %OSP" }, +[0x4d] = { 0,0, "DEC%S %OBP" }, +[0x4e] = { 0,0, "DEC%S %OSI" }, +[0x4f] = { 0,0, "DEC%S %ODI" }, +[0x50] = { 0,0, "PUSH%S %OAX" }, +[0x51] = { 0,0, "PUSH%S %OCX" }, +[0x52] = { 0,0, "PUSH%S %ODX" }, +[0x53] = { 0,0, "PUSH%S %OBX" }, +[0x54] = { 0,0, "PUSH%S %OSP" }, +[0x55] = { 0,0, "PUSH%S %OBP" }, +[0x56] = { 0,0, "PUSH%S %OSI" }, +[0x57] = { 0,0, "PUSH%S %ODI" }, +[0x58] = { 0,0, "POP%S %OAX" }, +[0x59] = { 0,0, "POP%S %OCX" }, +[0x5a] = { 0,0, "POP%S %ODX" }, +[0x5b] = { 0,0, "POP%S %OBX" }, +[0x5c] = { 0,0, "POP%S %OSP" }, +[0x5d] = { 0,0, "POP%S %OBP" }, +[0x5e] = { 0,0, "POP%S %OSI" }, +[0x5f] = { 0,0, "POP%S %ODI" }, +[0x60] = { 0,0, "PUSHA%S" }, +[0x61] = { 0,0, "POPA%S" }, +[0x62] = { RMM,0, "BOUND %e,%r" }, +[0x63] = { RM,0, "ARPL %r,%e" }, +[0x64] = { SEG,0, "FS:" }, +[0x65] = { SEG,0, "GS:" }, +[0x66] = { OPOVER,0, "" }, +[0x67] = { ADDOVER,0, "" }, +[0x68] = { Iwd,0, "PUSH%S %i" }, +[0x69] = { RM,Iwd, "IMUL%S %e,%i,%r" }, +[0x6a] = { Ib,0, "PUSH%S %i" }, +[0x6b] = { RM,Ibs, "IMUL%S %e,%i,%r" }, +[0x6c] = { 0,0, "INSB DX,(%ODI)" }, +[0x6d] = { 0,0, "INS%S DX,(%ODI)" }, +[0x6e] = { 0,0, "OUTSB (%ASI),DX" }, +[0x6f] = { 0,0, "OUTS%S (%ASI),DX" }, +[0x70] = { Jbs,0, "JOS %p" }, +[0x71] = { Jbs,0, "JOC %p" }, +[0x72] = { Jbs,0, "JCS %p" }, +[0x73] = { Jbs,0, "JCC %p" }, +[0x74] = { Jbs,0, "JEQ %p" }, +[0x75] = { Jbs,0, "JNE %p" }, +[0x76] = { Jbs,0, "JLS %p" }, +[0x77] = { Jbs,0, "JHI %p" }, +[0x78] = { Jbs,0, "JMI %p" }, +[0x79] = { Jbs,0, "JPL %p" }, +[0x7a] = { Jbs,0, "JPS %p" }, +[0x7b] = { Jbs,0, "JPC %p" }, +[0x7c] = { Jbs,0, "JLT %p" }, +[0x7d] = { Jbs,0, "JGE %p" }, +[0x7e] = { Jbs,0, "JLE %p" }, +[0x7f] = { Jbs,0, "JGT %p" }, +[0x80] = { RMOPB,0, optab80 }, +[0x81] = { RMOP,0, optab81 }, +[0x83] = { RMOP,0, optab83 }, +[0x84] = { RMB,0, "TESTB %r,%e" }, +[0x85] = { RM,0, "TEST%S %r,%e" }, +[0x86] = { RMB,0, "XCHGB %r,%e" }, +[0x87] = { RM,0, "XCHG%S %r,%e" }, +[0x88] = { RMB,0, "MOVB %r,%e" }, +[0x89] = { RM,0, "MOV%S %r,%e" }, +[0x8a] = { RMB,0, "MOVB %e,%r" }, +[0x8b] = { RM,0, "MOV%S %e,%r" }, +[0x8c] = { RM,0, "MOVW %g,%e" }, +[0x8d] = { RM,0, "LEA%S %e,%r" }, +[0x8e] = { RM,0, "MOVW %e,%g" }, +[0x8f] = { RM,0, "POP%S %e" }, +[0x90] = { 0,0, "NOP" }, +[0x91] = { 0,0, "XCHG %OCX,%OAX" }, +[0x92] = { 0,0, "XCHG %ODX,%OAX" }, +[0x93] = { 0,0, "XCHG %OBX,%OAX" }, +[0x94] = { 0,0, "XCHG %OSP,%OAX" }, +[0x95] = { 0,0, "XCHG %OBP,%OAX" }, +[0x96] = { 0,0, "XCHG %OSI,%OAX" }, +[0x97] = { 0,0, "XCHG %ODI,%OAX" }, +[0x98] = { 0,0, "%W" }, /* miserable CBW or CWDE */ +[0x99] = { 0,0, "%w" }, /* idiotic CWD or CDQ */ +[0x9a] = { PTR,0, "CALL%S %d" }, +[0x9b] = { 0,0, "WAIT" }, +[0x9c] = { 0,0, "PUSHF" }, +[0x9d] = { 0,0, "POPF" }, +[0x9e] = { 0,0, "SAHF" }, +[0x9f] = { 0,0, "LAHF" }, +[0xa0] = { Awd,0, "MOVB %i,AL" }, +[0xa1] = { Awd,0, "MOV%S %i,%OAX" }, +[0xa2] = { Awd,0, "MOVB AL,%i" }, +[0xa3] = { Awd,0, "MOV%S %OAX,%i" }, +[0xa4] = { 0,0, "MOVSB (%ASI),(%ADI)" }, +[0xa5] = { 0,0, "MOVS%S (%ASI),(%ADI)" }, +[0xa6] = { 0,0, "CMPSB (%ASI),(%ADI)" }, +[0xa7] = { 0,0, "CMPS%S (%ASI),(%ADI)" }, +[0xa8] = { Ib,0, "TESTB %i,AL" }, +[0xa9] = { Iwd,0, "TEST%S %i,%OAX" }, +[0xaa] = { 0,0, "STOSB AL,(%ADI)" }, +[0xab] = { 0,0, "STOS%S %OAX,(%ADI)" }, +[0xac] = { 0,0, "LODSB (%ASI),AL" }, +[0xad] = { 0,0, "LODS%S (%ASI),%OAX" }, +[0xae] = { 0,0, "SCASB (%ADI),AL" }, +[0xaf] = { 0,0, "SCAS%S (%ADI),%OAX" }, +[0xb0] = { Ib,0, "MOVB %i,AL" }, +[0xb1] = { Ib,0, "MOVB %i,CL" }, +[0xb2] = { Ib,0, "MOVB %i,DL" }, +[0xb3] = { Ib,0, "MOVB %i,BL" }, +[0xb4] = { Ib,0, "MOVB %i,AH" }, +[0xb5] = { Ib,0, "MOVB %i,CH" }, +[0xb6] = { Ib,0, "MOVB %i,DH" }, +[0xb7] = { Ib,0, "MOVB %i,BH" }, +[0xb8] = { Iwdq,0, "MOV%S %i,%OAX" }, +[0xb9] = { Iwdq,0, "MOV%S %i,%OCX" }, +[0xba] = { Iwdq,0, "MOV%S %i,%ODX" }, +[0xbb] = { Iwdq,0, "MOV%S %i,%OBX" }, +[0xbc] = { Iwdq,0, "MOV%S %i,%OSP" }, +[0xbd] = { Iwdq,0, "MOV%S %i,%OBP" }, +[0xbe] = { Iwdq,0, "MOV%S %i,%OSI" }, +[0xbf] = { Iwdq,0, "MOV%S %i,%ODI" }, +[0xc0] = { RMOPB,0, optabC0 }, +[0xc1] = { RMOP,0, optabC1 }, +[0xc2] = { Iw,0, "RET %i" }, +[0xc3] = { RET,0, "RET" }, +[0xc4] = { RM,0, "LES %e,%r" }, +[0xc5] = { RM,0, "LDS %e,%r" }, +[0xc6] = { RMB,Ib, "MOVB %i,%e" }, +[0xc7] = { RM,Iwd, "MOV%S %i,%e" }, +[0xc8] = { Iw2,Ib, "ENTER %i,%I" }, /* loony ENTER */ +[0xc9] = { RET,0, "LEAVE" }, /* bizarre LEAVE */ +[0xca] = { Iw,0, "RETF %i" }, +[0xcb] = { RET,0, "RETF" }, +[0xcc] = { 0,0, "INT 3" }, +[0xcd] = { Ib,0, "INTB %i" }, +[0xce] = { 0,0, "INTO" }, +[0xcf] = { 0,0, "IRET" }, +[0xd0] = { RMOPB,0, optabD0 }, +[0xd1] = { RMOP,0, optabD1 }, +[0xd2] = { RMOPB,0, optabD2 }, +[0xd3] = { RMOP,0, optabD3 }, +[0xd4] = { OA,0, "AAM" }, +[0xd5] = { OA,0, "AAD" }, +[0xd7] = { 0,0, "XLAT" }, +[0xd8] = { FRMOP,0, optabD8 }, +[0xd9] = { FRMEX,0, optabD9 }, +[0xda] = { FRMOP,0, optabDA }, +[0xdb] = { FRMEX,0, optabDB }, +[0xdc] = { FRMOP,0, optabDC }, +[0xdd] = { FRMOP,0, optabDD }, +[0xde] = { FRMOP,0, optabDE }, +[0xdf] = { FRMOP,0, optabDF }, +[0xe0] = { Jbs,0, "LOOPNE %p" }, +[0xe1] = { Jbs,0, "LOOPE %p" }, +[0xe2] = { Jbs,0, "LOOP %p" }, +[0xe3] = { Jbs,0, "JCXZ %p" }, +[0xe4] = { Ib,0, "INB %i,AL" }, +[0xe5] = { Ib,0, "IN%S %i,%OAX" }, +[0xe6] = { Ib,0, "OUTB AL,%i" }, +[0xe7] = { Ib,0, "OUT%S %OAX,%i" }, +[0xe8] = { Iwds,0, "CALL %p" }, +[0xe9] = { Iwds,0, "JMP %p" }, +[0xea] = { PTR,0, "JMP %d" }, +[0xeb] = { Jbs,0, "JMP %p" }, +[0xec] = { 0,0, "INB DX,AL" }, +[0xed] = { 0,0, "IN%S DX,%OAX" }, +[0xee] = { 0,0, "OUTB AL,DX" }, +[0xef] = { 0,0, "OUT%S %OAX,DX" }, +[0xf0] = { PRE,0, "LOCK" }, +[0xf2] = { OPRE,0, "REPNE" }, +[0xf3] = { OPRE,0, "REP" }, +[0xf4] = { 0,0, "HLT" }, +[0xf5] = { 0,0, "CMC" }, +[0xf6] = { RMOPB,0, optabF6 }, +[0xf7] = { RMOP,0, optabF7 }, +[0xf8] = { 0,0, "CLC" }, +[0xf9] = { 0,0, "STC" }, +[0xfa] = { 0,0, "CLI" }, +[0xfb] = { 0,0, "STI" }, +[0xfc] = { 0,0, "CLD" }, +[0xfd] = { 0,0, "STD" }, +[0xfe] = { RMOPB,0, optabFE }, +[0xff] = { RMOP,0, optabFF }, +[0x100] = { RM,0, "MOVLQSX %e,%r" }, +[0x101] = { RM,0, "MOVLQZX %e,%r" }, }; /* @@ -1894,24 +1894,24 @@ bprint(Instr *ip, char *fmt, ...) #define ONAME(ip) "" static char *reg[] = { -[AX] "AX", -[CX] "CX", -[DX] "DX", -[BX] "BX", -[SP] "SP", -[BP] "BP", -[SI] "SI", -[DI] "DI", +[AX] = "AX", +[CX] = "CX", +[DX] = "DX", +[BX] = "BX", +[SP] = "SP", +[BP] = "BP", +[SI] = "SI", +[DI] = "DI", /* amd64 */ -[AMD64_R8] "R8", -[AMD64_R9] "R9", -[AMD64_R10] "R10", -[AMD64_R11] "R11", -[AMD64_R12] "R12", -[AMD64_R13] "R13", -[AMD64_R14] "R14", -[AMD64_R15] "R15", +[AMD64_R8] = "R8", +[AMD64_R9] = "R9", +[AMD64_R10] = "R10", +[AMD64_R11] = "R11", +[AMD64_R12] = "R12", +[AMD64_R13] = "R13", +[AMD64_R14] = "R14", +[AMD64_R15] = "R15", }; static char *breg[] = { "AL", "CL", "DL", "BL", "AH", "CH", "DH", "BH" }; diff --git a/src/libmach/darwin.c b/src/libmach/darwin.c index d44fd5612..c443a4fba 100644 --- a/src/libmach/darwin.c +++ b/src/libmach/darwin.c @@ -170,7 +170,7 @@ me(kern_return_t r) // handles to tasks (processes), and handles to threads within a // process. All of them are small integers. // -// To accomodate Mach, we employ a clumsy hack: in this interface, +// To accommodate Mach, we employ a clumsy hack: in this interface, // if you pass in a positive number, that's a process id. // If you pass in a negative number, that identifies a thread that // has been previously returned by procthreadpids (it indexes diff --git a/src/libmach/obj.c b/src/libmach/obj.c index 7d660787b..7999f24c6 100644 --- a/src/libmach/obj.c +++ b/src/libmach/obj.c @@ -81,17 +81,17 @@ struct Obj /* functions to handle each intermediate (.$O) file */ static Obj obj[] = { /* functions to identify and parse each type of obj */ - [Obj68020] "68020 .2", _is2, _read2, - [ObjAmd64] "amd64 .6", _is6, _read6, - [ObjArm] "arm .5", _is5, _read5, - [ObjAlpha] "alpha .7", _is7, _read7, - [Obj386] "386 .8", _is8, _read8, - [ObjSparc] "sparc .k", _isk, _readk, - [ObjPower] "power .q", _isq, _readq, - [ObjMips] "mips .v", _isv, _readv, - [ObjSparc64] "sparc64 .u", _isu, _readu, - [ObjPower64] "power64 .9", _is9, _read9, - [Maxobjtype] 0, 0 + [Obj68020] = { "68020 .2", _is2, _read2 }, + [ObjAmd64] = { "amd64 .6", _is6 , _read6 }, + [ObjArm] = { "arm .5", _is5, _read5 }, + [ObjAlpha] = { "alpha .7", _is7, _read7 }, + [Obj386] = { "386 .8", _is8, _read8 }, + [ObjSparc] = { "sparc .k", _isk, _readk }, + [ObjPower] = { "power .q", _isq, _readq }, + [ObjMips] = { "mips .v", _isv, _readv }, + [ObjSparc64] = { "sparc64 .u", _isu, _readu }, + [ObjPower64] = { "power64 .9", _is9, _read9 }, + [Maxobjtype] = { 0, 0, 0 } }; struct Symtab diff --git a/src/make.bash b/src/make.bash index 79e368cb5..84b9908f4 100755 --- a/src/make.bash +++ b/src/make.bash @@ -47,6 +47,9 @@ rm -f "$GOBIN"/gomake ) >"$GOBIN"/gomake chmod +x "$GOBIN"/gomake +# TODO(brainman): delete this after 01/01/2012. +rm -f "$GOBIN"/gotest # remove old bash version of gotest on Windows + if [ -d /selinux -a -f /selinux/booleans/allow_execstack -a -x /usr/sbin/selinuxenabled ] && /usr/sbin/selinuxenabled; then if ! cat /selinux/booleans/allow_execstack | grep -c '^1 1$' >> /dev/null ; then echo "WARNING: the default SELinux policy on, at least, Fedora 12 breaks " diff --git a/src/pkg/Makefile b/src/pkg/Makefile index b046064a6..2d6b3d014 100644 --- a/src/pkg/Makefile +++ b/src/pkg/Makefile @@ -59,6 +59,7 @@ DIRS=\ crypto/tls\ crypto/twofish\ crypto/x509\ + crypto/x509/pkix\ crypto/xtea\ debug/dwarf\ debug/macho\ @@ -73,17 +74,17 @@ DIRS=\ encoding/binary\ encoding/git85\ encoding/hex\ - encoding/line\ encoding/pem\ exec\ exp/datafmt\ - exp/draw\ - exp/draw/x11\ exp/eval\ + exp/gui\ + exp/gui/x11\ expvar\ flag\ fmt\ go/ast\ + go/build\ go/doc\ go/parser\ go/printer\ @@ -103,15 +104,21 @@ DIRS=\ http/fcgi\ http/pprof\ http/httptest\ + http/spdy\ image\ + image/bmp\ + image/draw\ + image/gif\ image/jpeg\ image/png\ + image/tiff\ image/ycbcr\ index/suffixarray\ io\ io/ioutil\ json\ log\ + mail\ math\ mime\ mime/multipart\ @@ -177,14 +184,18 @@ endif NOTEST+=\ crypto\ crypto/openpgp/error\ + crypto/x509/pkix\ debug/proc\ - exp/draw/x11\ + exp/gui\ + exp/gui/x11\ go/ast\ go/doc\ go/token\ hash\ http/pprof\ http/httptest\ + image/bmp\ + image/gif\ net/dict\ rand\ runtime/cgo\ diff --git a/src/pkg/asn1/asn1.go b/src/pkg/asn1/asn1.go index 5f470aed7..2650ef2a2 100644 --- a/src/pkg/asn1/asn1.go +++ b/src/pkg/asn1/asn1.go @@ -20,6 +20,7 @@ package asn1 // everything by any means. import ( + "big" "fmt" "os" "reflect" @@ -88,6 +89,27 @@ func parseInt(bytes []byte) (int, os.Error) { return int(ret64), nil } +var bigOne = big.NewInt(1) + +// parseBigInt treats the given bytes as a big-endian, signed integer and returns +// the result. +func parseBigInt(bytes []byte) *big.Int { + ret := new(big.Int) + if len(bytes) > 0 && bytes[0]&0x80 == 0x80 { + // This is a negative number. + notBytes := make([]byte, len(bytes)) + for i := range notBytes { + notBytes[i] = ^bytes[i] + } + ret.SetBytes(notBytes) + ret.Add(ret, bigOne) + ret.Neg(ret) + return ret + } + ret.SetBytes(bytes) + return ret +} + // BIT STRING // BitString is the structure to use when you want an ASN.1 BIT STRING type. A @@ -164,9 +186,9 @@ func (oi ObjectIdentifier) Equal(other ObjectIdentifier) bool { return true } -// parseObjectIdentifier parses an OBJECT IDENTIFER from the given bytes and -// returns it. An object identifer is a sequence of variable length integers -// that are assigned in a hierarachy. +// parseObjectIdentifier parses an OBJECT IDENTIFIER from the given bytes and +// returns it. An object identifier is a sequence of variable length integers +// that are assigned in a hierarchy. func parseObjectIdentifier(bytes []byte) (s []int, err os.Error) { if len(bytes) == 0 { err = SyntaxError{"zero length OBJECT IDENTIFIER"} @@ -269,7 +291,7 @@ func isPrintable(b byte) bool { b == ':' || b == '=' || b == '?' || - // This is techincally not allowed in a PrintableString. + // This is technically not allowed in a PrintableString. // However, x509 certificates with wildcard strings don't // always use the correct string type so we permit it. b == '*' @@ -425,6 +447,7 @@ var ( timeType = reflect.TypeOf(&time.Time{}) rawValueType = reflect.TypeOf(RawValue{}) rawContentsType = reflect.TypeOf(RawContent(nil)) + bigIntType = reflect.TypeOf(new(big.Int)) ) // invalidLength returns true iff offset + length > sliceLength, or if the @@ -639,6 +662,10 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam case flagType: v.SetBool(true) return + case bigIntType: + parsedInt := parseBigInt(innerBytes) + v.Set(reflect.ValueOf(parsedInt)) + return } switch val := v; val.Kind() { case reflect.Bool: diff --git a/src/pkg/asn1/asn1_test.go b/src/pkg/asn1/asn1_test.go index 78f562805..463dbe026 100644 --- a/src/pkg/asn1/asn1_test.go +++ b/src/pkg/asn1/asn1_test.go @@ -42,6 +42,33 @@ func TestParseInt64(t *testing.T) { } } +var bigIntTests = []struct { + in []byte + base10 string +}{ + {[]byte{0xff}, "-1"}, + {[]byte{0x00}, "0"}, + {[]byte{0x01}, "1"}, + {[]byte{0x00, 0xff}, "255"}, + {[]byte{0xff, 0x00}, "-256"}, + {[]byte{0x01, 0x00}, "256"}, +} + +func TestParseBigInt(t *testing.T) { + for i, test := range bigIntTests { + ret := parseBigInt(test.in) + if ret.String() != test.base10 { + t.Errorf("#%d: bad result from %x, got %s want %s", i, test.in, ret.String(), test.base10) + } + fw := newForkableWriter() + marshalBigInt(fw, ret) + result := fw.Bytes() + if !bytes.Equal(result, test.in) { + t.Errorf("#%d: got %x from marshaling %s, want %x", i, result, ret, test.in) + } + } +} + type bitStringTest struct { in []byte ok bool diff --git a/src/pkg/asn1/common.go b/src/pkg/asn1/common.go index 158987747..9db887e25 100644 --- a/src/pkg/asn1/common.go +++ b/src/pkg/asn1/common.go @@ -10,7 +10,7 @@ import ( "strings" ) -// ASN.1 objects have metadata preceeding them: +// ASN.1 objects have metadata preceding them: // the tag: the type of the object // a flag denoting if this object is compound or not // the class type: the namespace of the tag @@ -132,6 +132,8 @@ func getUniversalType(t reflect.Type) (tagNumber int, isCompound, ok bool) { return tagUTCTime, false, true case enumeratedType: return tagEnum, false, true + case bigIntType: + return tagInteger, false, true } switch t.Kind() { case reflect.Bool: diff --git a/src/pkg/asn1/marshal.go b/src/pkg/asn1/marshal.go index a3e1145b8..7212c91ef 100644 --- a/src/pkg/asn1/marshal.go +++ b/src/pkg/asn1/marshal.go @@ -5,6 +5,7 @@ package asn1 import ( + "big" "bytes" "fmt" "io" @@ -125,6 +126,43 @@ func int64Length(i int64) (numBytes int) { return } +func marshalBigInt(out *forkableWriter, n *big.Int) (err os.Error) { + if n.Sign() < 0 { + // A negative number has to be converted to two's-complement + // form. So we'll subtract 1 and invert. If the + // most-significant-bit isn't set then we'll need to pad the + // beginning with 0xff in order to keep the number negative. + nMinus1 := new(big.Int).Neg(n) + nMinus1.Sub(nMinus1, bigOne) + bytes := nMinus1.Bytes() + for i := range bytes { + bytes[i] ^= 0xff + } + if len(bytes) == 0 || bytes[0]&0x80 == 0 { + err = out.WriteByte(0xff) + if err != nil { + return + } + } + _, err = out.Write(bytes) + } else if n.Sign() == 0 { + // Zero is written as a single 0 zero rather than no bytes. + err = out.WriteByte(0x00) + } else { + bytes := n.Bytes() + if len(bytes) > 0 && bytes[0]&0x80 != 0 { + // We'll have to pad this with 0x00 in order to stop it + // looking like a negative number. + err = out.WriteByte(0) + if err != nil { + return + } + } + _, err = out.Write(bytes) + } + return +} + func marshalLength(out *forkableWriter, i int) (err os.Error) { n := lengthLength(i) @@ -334,6 +372,8 @@ func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameter return marshalBitString(out, value.Interface().(BitString)) case objectIdentifierType: return marshalObjectIdentifier(out, value.Interface().(ObjectIdentifier)) + case bigIntType: + return marshalBigInt(out, value.Interface().(*big.Int)) } switch v := value; v.Kind() { @@ -351,7 +391,7 @@ func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameter startingField := 0 // If the first element of the structure is a non-empty - // RawContents, then we don't bother serialising the rest. + // RawContents, then we don't bother serializing the rest. if t.NumField() > 0 && t.Field(0).Type == rawContentsType { s := v.Field(0) if s.Len() > 0 { @@ -361,7 +401,7 @@ func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameter } /* The RawContents will contain the tag and * length fields but we'll also be writing - * those outselves, so we strip them out of + * those ourselves, so we strip them out of * bytes */ _, err = out.Write(stripTagAndLength(bytes)) return @@ -418,6 +458,10 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) return marshalField(out, v.Elem(), params) } + if params.optional && reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface()) { + return + } + if v.Type() == rawValueType { rv := v.Interface().(RawValue) err = marshalTagAndLength(out, tagAndLength{rv.Class, rv.Tag, len(rv.Bytes), rv.IsCompound}) @@ -428,10 +472,6 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) return } - if params.optional && reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface()) { - return - } - tag, isCompound, ok := getUniversalType(v.Type()) if !ok { err = StructuralError{fmt.Sprintf("unknown Go type: %v", v.Type())} diff --git a/src/pkg/asn1/marshal_test.go b/src/pkg/asn1/marshal_test.go index cd165d203..a9517634d 100644 --- a/src/pkg/asn1/marshal_test.go +++ b/src/pkg/asn1/marshal_test.go @@ -45,6 +45,10 @@ type printableStringTest struct { A string "printable" } +type optionalRawValueTest struct { + A RawValue "optional" +} + type testSET []int func setPST(t *time.Time) *time.Time { @@ -102,6 +106,7 @@ var marshalTests = []marshalTest{ "7878787878787878787878787878787878787878787878787878787878787878", }, {ia5StringTest{"test"}, "3006160474657374"}, + {optionalRawValueTest{}, "3000"}, {printableStringTest{"test"}, "3006130474657374"}, {printableStringTest{"test*"}, "30071305746573742a"}, {rawContentsStruct{nil, 64}, "3003020140"}, diff --git a/src/pkg/big/arith.go b/src/pkg/big/arith.go index df3808f5e..b9aeea968 100644 --- a/src/pkg/big/arith.go +++ b/src/pkg/big/arith.go @@ -188,7 +188,7 @@ func subVW_g(z, x []Word, y Word) (c Word) { } -func shlVW_g(z, x []Word, s Word) (c Word) { +func shlVU_g(z, x []Word, s uint) (c Word) { if n := len(z); n > 0 { ŝ := _W - s w1 := x[n-1] @@ -204,7 +204,7 @@ func shlVW_g(z, x []Word, s Word) (c Word) { } -func shrVW_g(z, x []Word, s Word) (c Word) { +func shrVU_g(z, x []Word, s uint) (c Word) { if n := len(z); n > 0 { ŝ := _W - s w1 := x[0] diff --git a/src/pkg/big/arith_386.s b/src/pkg/big/arith_386.s index 21521635b..07c07b02c 100644 --- a/src/pkg/big/arith_386.s +++ b/src/pkg/big/arith_386.s @@ -118,8 +118,8 @@ E4: CMPL BX, BP // i < n RET -// func shlVW(z, x []Word, s Word) (c Word) -TEXT ·shlVW(SB),7,$0 +// func shlVU(z, x []Word, s uint) (c Word) +TEXT ·shlVU(SB),7,$0 MOVL n+4(FP), BX // i = n SUBL $1, BX // i-- JL X8b // i < 0 (n <= 0) @@ -153,8 +153,8 @@ X8b: MOVL $0, c+28(FP) RET -// func shrVW(z, x []Word, s Word) (c Word) -TEXT ·shrVW(SB),7,$0 +// func shrVU(z, x []Word, s uint) (c Word) +TEXT ·shrVU(SB),7,$0 MOVL n+4(FP), BP SUBL $1, BP // n-- JL X9b // n < 0 (n <= 0) diff --git a/src/pkg/big/arith_amd64.s b/src/pkg/big/arith_amd64.s index c740565a7..89b65f38a 100644 --- a/src/pkg/big/arith_amd64.s +++ b/src/pkg/big/arith_amd64.s @@ -120,8 +120,8 @@ E4: CMPQ BX, R11 // i < n RET -// func shlVW(z, x []Word, s Word) (c Word) -TEXT ·shlVW(SB),7,$0 +// func shlVU(z, x []Word, s uint) (c Word) +TEXT ·shlVU(SB),7,$0 MOVL n+8(FP), BX // i = n SUBL $1, BX // i-- JL X8b // i < 0 (n <= 0) @@ -129,7 +129,7 @@ TEXT ·shlVW(SB),7,$0 // n > 0 MOVQ z+0(FP), R10 MOVQ x+16(FP), R8 - MOVQ s+32(FP), CX + MOVL s+32(FP), CX MOVQ (R8)(BX*8), AX // w1 = x[n-1] MOVQ $0, DX SHLQ CX, DX:AX // w1>>ŝ @@ -155,8 +155,8 @@ X8b: MOVQ $0, c+40(FP) RET -// func shrVW(z, x []Word, s Word) (c Word) -TEXT ·shrVW(SB),7,$0 +// func shrVU(z, x []Word, s uint) (c Word) +TEXT ·shrVU(SB),7,$0 MOVL n+8(FP), R11 SUBL $1, R11 // n-- JL X9b // n < 0 (n <= 0) @@ -164,7 +164,7 @@ TEXT ·shrVW(SB),7,$0 // n > 0 MOVQ z+0(FP), R10 MOVQ x+16(FP), R8 - MOVQ s+32(FP), CX + MOVL s+32(FP), CX MOVQ (R8), AX // w1 = x[0] MOVQ $0, DX SHRQ CX, DX:AX // w1<<ŝ diff --git a/src/pkg/big/arith_arm.s b/src/pkg/big/arith_arm.s index e4a9a962c..60abe6eaa 100644 --- a/src/pkg/big/arith_arm.s +++ b/src/pkg/big/arith_arm.s @@ -97,6 +97,7 @@ E3: RET +// func subVW(z, x []Word, y Word) (c Word) TEXT ·subVW(SB),7,$0 MOVW z+0(FP), R1 MOVW x+12(FP), R2 @@ -131,8 +132,8 @@ E4: RET -// func shlVW(z, x []Word, s Word) (c Word) -TEXT ·shlVW(SB),7,$0 +// func shlVU(z, x []Word, s uint) (c Word) +TEXT ·shlVU(SB),7,$0 MOVW n+4(FP), R5 CMP $0, R5 BEQ X7 @@ -180,7 +181,8 @@ X7: RET -TEXT ·shrVW(SB),7,$0 +// func shrVU(z, x []Word, s uint) (c Word) +TEXT ·shrVU(SB),7,$0 MOVW n+4(FP), R5 CMP $0, R5 BEQ X6 @@ -229,6 +231,7 @@ X6: RET +// func mulAddVWW(z, x []Word, y, r Word) (c Word) TEXT ·mulAddVWW(SB),7,$0 MOVW $0, R0 MOVW z+0(FP), R1 @@ -256,6 +259,7 @@ E8: RET +// func addMulVVW(z, x []Word, y Word) (c Word) TEXT ·addMulVVW(SB),7,$0 MOVW $0, R0 MOVW z+0(FP), R1 @@ -286,11 +290,13 @@ E9: RET +// divWVW(z* Word, xn Word, x []Word, y Word) (r Word) TEXT ·divWVW(SB),7,$0 // ARM has no multiword division, so use portable code. B ·divWVW_g(SB) +// func divWW(x1, x0, y Word) (q, r Word) TEXT ·divWW(SB),7,$0 // ARM has no multiword division, so use portable code. B ·divWW_g(SB) diff --git a/src/pkg/big/arith_decl.go b/src/pkg/big/arith_decl.go index c456d5f67..95fcd8b94 100644 --- a/src/pkg/big/arith_decl.go +++ b/src/pkg/big/arith_decl.go @@ -11,8 +11,8 @@ func addVV(z, x, y []Word) (c Word) func subVV(z, x, y []Word) (c Word) func addVW(z, x []Word, y Word) (c Word) func subVW(z, x []Word, y Word) (c Word) -func shlVW(z, x []Word, s Word) (c Word) -func shrVW(z, x []Word, s Word) (c Word) +func shlVU(z, x []Word, s uint) (c Word) +func shrVU(z, x []Word, s uint) (c Word) func mulAddVWW(z, x []Word, y, r Word) (c Word) func addMulVVW(z, x []Word, y Word) (c Word) func divWVW(z []Word, xn Word, x []Word, y Word) (r Word) diff --git a/src/pkg/big/arith_test.go b/src/pkg/big/arith_test.go index 934b302df..f3e2d4735 100644 --- a/src/pkg/big/arith_test.go +++ b/src/pkg/big/arith_test.go @@ -185,6 +185,13 @@ func testFunVW(t *testing.T, msg string, f funVW, a argVW) { } +func makeFunVW(f func(z, x []Word, s uint) (c Word)) funVW { + return func(z, x []Word, s Word) (c Word) { + return f(z, x, uint(s)) + } +} + + func TestFunVW(t *testing.T) { for _, a := range sumVW { arg := a @@ -196,16 +203,20 @@ func TestFunVW(t *testing.T) { testFunVW(t, "subVW", subVW, arg) } + shlVW_g := makeFunVW(shlVU_g) + shlVW := makeFunVW(shlVU) for _, a := range lshVW { arg := a - testFunVW(t, "shlVW_g", shlVW_g, arg) - testFunVW(t, "shlVW", shlVW, arg) + testFunVW(t, "shlVU_g", shlVW_g, arg) + testFunVW(t, "shlVU", shlVW, arg) } + shrVW_g := makeFunVW(shrVU_g) + shrVW := makeFunVW(shrVU) for _, a := range rshVW { arg := a - testFunVW(t, "shrVW_g", shrVW_g, arg) - testFunVW(t, "shrVW", shrVW, arg) + testFunVW(t, "shrVU_g", shrVW_g, arg) + testFunVW(t, "shrVU", shrVW, arg) } } diff --git a/src/pkg/big/int.go b/src/pkg/big/int.go index f1ea7b1c2..22bdf8d2f 100755 --- a/src/pkg/big/int.go +++ b/src/pkg/big/int.go @@ -8,8 +8,10 @@ package big import ( "fmt" + "io" "os" "rand" + "strings" ) // An Int represents a signed multi-precision integer. @@ -309,47 +311,134 @@ func (x *Int) Cmp(y *Int) (r int) { func (x *Int) String() string { - s := "" - if x.neg { - s = "-" + switch { + case x == nil: + return "<nil>" + case x.neg: + return "-" + x.abs.decimalString() } - return s + x.abs.string(10) + return x.abs.decimalString() } -func fmtbase(ch int) int { +func charset(ch int) string { switch ch { case 'b': - return 2 + return lowercaseDigits[0:2] case 'o': - return 8 - case 'd': - return 10 + return lowercaseDigits[0:8] + case 'd', 's', 'v': + return lowercaseDigits[0:10] case 'x': - return 16 + return lowercaseDigits[0:16] + case 'X': + return uppercaseDigits[0:16] } - return 10 + return "" // unknown format } // Format is a support routine for fmt.Formatter. It accepts -// the formats 'b' (binary), 'o' (octal), 'd' (decimal) and -// 'x' (hexadecimal). +// the formats 'b' (binary), 'o' (octal), 'd' (decimal), 'x' +// (lowercase hexadecimal), and 'X' (uppercase hexadecimal). // func (x *Int) Format(s fmt.State, ch int) { - if x == nil { + cs := charset(ch) + + // special cases + switch { + case cs == "": + // unknown format + fmt.Fprintf(s, "%%!%c(big.Int=%s)", ch, x.String()) + return + case x == nil: fmt.Fprint(s, "<nil>") return } + + // determine format + format := "%s" + if s.Flag('#') { + switch ch { + case 'o': + format = "0%s" + case 'x': + format = "0x%s" + case 'X': + format = "0X%s" + } + } if x.neg { - fmt.Fprint(s, "-") + format = "-" + format + } + + fmt.Fprintf(s, format, x.abs.string(cs)) +} + + +// scan sets z to the integer value corresponding to the longest possible prefix +// read from r representing a signed integer number in a given conversion base. +// It returns z, the actual conversion base used, and an error, if any. In the +// error case, the value of z is undefined. The syntax follows the syntax of +// integer literals in Go. +// +// The base argument must be 0 or a value from 2 through MaxBase. If the base +// is 0, the string prefix determines the actual conversion base. A prefix of +// ``0x'' or ``0X'' selects base 16; the ``0'' prefix selects base 8, and a +// ``0b'' or ``0B'' prefix selects base 2. Otherwise the selected base is 10. +// +func (z *Int) scan(r io.RuneScanner, base int) (*Int, int, os.Error) { + // determine sign + ch, _, err := r.ReadRune() + if err != nil { + return z, 0, err + } + neg := false + switch ch { + case '-': + neg = true + case '+': // nothing to do + default: + r.UnreadRune() + } + + // determine mantissa + z.abs, base, err = z.abs.scan(r, base) + if err != nil { + return z, base, err + } + z.neg = len(z.abs) > 0 && neg // 0 has no sign + + return z, base, nil +} + + +// Scan is a support routine for fmt.Scanner; it sets z to the value of +// the scanned number. It accepts the formats 'b' (binary), 'o' (octal), +// 'd' (decimal), 'x' (lowercase hexadecimal), and 'X' (uppercase hexadecimal). +func (z *Int) Scan(s fmt.ScanState, ch int) os.Error { + base := 0 + switch ch { + case 'b': + base = 2 + case 'o': + base = 8 + case 'd': + base = 10 + case 'x', 'X': + base = 16 + case 's', 'v': + // let scan determine the base + default: + return os.ErrorString("Int.Scan: invalid verb") } - fmt.Fprint(s, x.abs.string(fmtbase(ch))) + _, _, err := z.scan(s, base) + return err } -// Int64 returns the int64 representation of z. -// If z cannot be represented in an int64, the result is undefined. +// Int64 returns the int64 representation of x. +// If x cannot be represented in an int64, the result is undefined. func (x *Int) Int64() int64 { if len(x.abs) == 0 { return 0 @@ -369,32 +458,19 @@ func (x *Int) Int64() int64 { // and returns z and a boolean indicating success. If SetString fails, // the value of z is undefined. // -// If the base argument is 0, the string prefix determines the actual -// conversion base. A prefix of ``0x'' or ``0X'' selects base 16; the -// ``0'' prefix selects base 8, and a ``0b'' or ``0B'' prefix selects -// base 2. Otherwise the selected base is 10. +// The base argument must be 0 or a value from 2 through MaxBase. If the base +// is 0, the string prefix determines the actual conversion base. A prefix of +// ``0x'' or ``0X'' selects base 16; the ``0'' prefix selects base 8, and a +// ``0b'' or ``0B'' prefix selects base 2. Otherwise the selected base is 10. // func (z *Int) SetString(s string, base int) (*Int, bool) { - if len(s) == 0 || base < 0 || base == 1 || 16 < base { + r := strings.NewReader(s) + _, _, err := z.scan(r, base) + if err != nil { return z, false } - - neg := s[0] == '-' - if neg || s[0] == '+' { - s = s[1:] - if len(s) == 0 { - return z, false - } - } - - var scanned int - z.abs, _, scanned = z.abs.scan(s, base) - if scanned != len(s) { - return z, false - } - z.neg = len(z.abs) > 0 && neg // 0 has no sign - - return z, true + _, _, err = r.ReadRune() + return z, err == os.EOF // err == os.EOF => scan consumed all of s } @@ -560,6 +636,42 @@ func (z *Int) Rsh(x *Int, n uint) *Int { } +// Bit returns the value of the i'th bit of z. That is, it +// returns (z>>i)&1. The bit index i must be >= 0. +func (z *Int) Bit(i int) uint { + if i < 0 { + panic("negative bit index") + } + if z.neg { + t := nat{}.sub(z.abs, natOne) + return t.bit(uint(i)) ^ 1 + } + + return z.abs.bit(uint(i)) +} + + +// SetBit sets the i'th bit of z to bit and returns z. +// That is, if bit is 1 SetBit sets z = x | (1 << i); +// if bit is 0 it sets z = x &^ (1 << i). If bit is not 0 or 1, +// SetBit will panic. +func (z *Int) SetBit(x *Int, i int, b uint) *Int { + if i < 0 { + panic("negative bit index") + } + if x.neg { + t := z.abs.sub(x.abs, natOne) + t = t.setBit(t, uint(i), b^1) + z.abs = t.add(t, natOne) + z.neg = len(z.abs) > 0 + return z + } + z.abs = z.abs.setBit(x.abs, uint(i), b) + z.neg = false + return z +} + + // And sets z = x & y and returns z. func (z *Int) And(x, y *Int) *Int { if x.neg == y.neg { @@ -704,13 +816,13 @@ func (z *Int) Not(x *Int) *Int { // Gob codec version. Permits backward-compatible changes to the encoding. -const version byte = 1 +const intGobVersion byte = 1 // GobEncode implements the gob.GobEncoder interface. func (z *Int) GobEncode() ([]byte, os.Error) { - buf := make([]byte, len(z.abs)*_S+1) // extra byte for version and sign bit + buf := make([]byte, 1+len(z.abs)*_S) // extra byte for version and sign bit i := z.abs.bytes(buf) - 1 // i >= 0 - b := version << 1 // make space for sign bit + b := intGobVersion << 1 // make space for sign bit if z.neg { b |= 1 } @@ -722,11 +834,11 @@ func (z *Int) GobEncode() ([]byte, os.Error) { // GobDecode implements the gob.GobDecoder interface. func (z *Int) GobDecode(buf []byte) os.Error { if len(buf) == 0 { - return os.NewError("Int.GobDecode: no data") + return os.ErrorString("Int.GobDecode: no data") } b := buf[0] - if b>>1 != version { - return os.NewError(fmt.Sprintf("Int.GobDecode: encoding version %d not supported", b>>1)) + if b>>1 != intGobVersion { + return os.ErrorString(fmt.Sprintf("Int.GobDecode: encoding version %d not supported", b>>1)) } z.neg = b&1 != 0 z.abs = z.abs.setBytes(buf[1:]) diff --git a/src/pkg/big/int_test.go b/src/pkg/big/int_test.go index 9c19dd5da..58a55030d 100755 --- a/src/pkg/big/int_test.go +++ b/src/pkg/big/int_test.go @@ -348,6 +348,98 @@ func TestSetString(t *testing.T) { } +var formatTests = []struct { + input string + format string + output string +}{ + {"<nil>", "%x", "<nil>"}, + {"<nil>", "%#x", "<nil>"}, + {"<nil>", "%#y", "%!y(big.Int=<nil>)"}, + + {"10", "%b", "1010"}, + {"10", "%o", "12"}, + {"10", "%d", "10"}, + {"10", "%v", "10"}, + {"10", "%x", "a"}, + {"10", "%X", "A"}, + {"-10", "%X", "-A"}, + {"10", "%y", "%!y(big.Int=10)"}, + {"-10", "%y", "%!y(big.Int=-10)"}, + + {"10", "%#b", "1010"}, + {"10", "%#o", "012"}, + {"10", "%#d", "10"}, + {"10", "%#v", "10"}, + {"10", "%#x", "0xa"}, + {"10", "%#X", "0XA"}, + {"-10", "%#X", "-0XA"}, + {"10", "%#y", "%!y(big.Int=10)"}, + {"-10", "%#y", "%!y(big.Int=-10)"}, +} + + +func TestFormat(t *testing.T) { + for i, test := range formatTests { + var x *Int + if test.input != "<nil>" { + var ok bool + x, ok = new(Int).SetString(test.input, 0) + if !ok { + t.Errorf("#%d failed reading input %s", i, test.input) + } + } + output := fmt.Sprintf(test.format, x) + if output != test.output { + t.Errorf("#%d got %s; want %s", i, output, test.output) + } + } +} + + +var scanTests = []struct { + input string + format string + output string + remaining int +}{ + {"1010", "%b", "10", 0}, + {"0b1010", "%v", "10", 0}, + {"12", "%o", "10", 0}, + {"012", "%v", "10", 0}, + {"10", "%d", "10", 0}, + {"10", "%v", "10", 0}, + {"a", "%x", "10", 0}, + {"0xa", "%v", "10", 0}, + {"A", "%X", "10", 0}, + {"-A", "%X", "-10", 0}, + {"+0b1011001", "%v", "89", 0}, + {"0xA", "%v", "10", 0}, + {"0 ", "%v", "0", 1}, + {"2+3", "%v", "2", 2}, + {"0XABC 12", "%v", "2748", 3}, +} + + +func TestScan(t *testing.T) { + var buf bytes.Buffer + for i, test := range scanTests { + x := new(Int) + buf.Reset() + buf.WriteString(test.input) + if _, err := fmt.Fscanf(&buf, test.format, x); err != nil { + t.Errorf("#%d error: %s", i, err.String()) + } + if x.String() != test.output { + t.Errorf("#%d got %s; want %s", i, x.String(), test.output) + } + if buf.Len() != test.remaining { + t.Errorf("#%d got %d bytes remaining; want %d", i, buf.Len(), test.remaining) + } + } +} + + // Examples from the Go Language Spec, section "Arithmetic operators" var divisionSignsTests = []struct { x, y int64 @@ -985,6 +1077,152 @@ func testBitFunSelf(t *testing.T, msg string, f bitFun, x, y *Int, exp string) { } +func altBit(x *Int, i int) uint { + z := new(Int).Rsh(x, uint(i)) + z = z.And(z, NewInt(1)) + if z.Cmp(new(Int)) != 0 { + return 1 + } + return 0 +} + + +func altSetBit(z *Int, x *Int, i int, b uint) *Int { + one := NewInt(1) + m := one.Lsh(one, uint(i)) + switch b { + case 1: + return z.Or(x, m) + case 0: + return z.AndNot(x, m) + } + panic("set bit is not 0 or 1") +} + + +func testBitset(t *testing.T, x *Int) { + n := x.BitLen() + z := new(Int).Set(x) + z1 := new(Int).Set(x) + for i := 0; i < n+10; i++ { + old := z.Bit(i) + old1 := altBit(z1, i) + if old != old1 { + t.Errorf("bitset: inconsistent value for Bit(%s, %d), got %v want %v", z1, i, old, old1) + } + z := new(Int).SetBit(z, i, 1) + z1 := altSetBit(new(Int), z1, i, 1) + if z.Bit(i) == 0 { + t.Errorf("bitset: bit %d of %s got 0 want 1", i, x) + } + if z.Cmp(z1) != 0 { + t.Errorf("bitset: inconsistent value after SetBit 1, got %s want %s", z, z1) + } + z.SetBit(z, i, 0) + altSetBit(z1, z1, i, 0) + if z.Bit(i) != 0 { + t.Errorf("bitset: bit %d of %s got 1 want 0", i, x) + } + if z.Cmp(z1) != 0 { + t.Errorf("bitset: inconsistent value after SetBit 0, got %s want %s", z, z1) + } + altSetBit(z1, z1, i, old) + z.SetBit(z, i, old) + if z.Cmp(z1) != 0 { + t.Errorf("bitset: inconsistent value after SetBit old, got %s want %s", z, z1) + } + } + if z.Cmp(x) != 0 { + t.Errorf("bitset: got %s want %s", z, x) + } +} + + +var bitsetTests = []struct { + x string + i int + b uint +}{ + {"0", 0, 0}, + {"0", 200, 0}, + {"1", 0, 1}, + {"1", 1, 0}, + {"-1", 0, 1}, + {"-1", 200, 1}, + {"0x2000000000000000000000000000", 108, 0}, + {"0x2000000000000000000000000000", 109, 1}, + {"0x2000000000000000000000000000", 110, 0}, + {"-0x2000000000000000000000000001", 108, 1}, + {"-0x2000000000000000000000000001", 109, 0}, + {"-0x2000000000000000000000000001", 110, 1}, +} + + +func TestBitSet(t *testing.T) { + for _, test := range bitwiseTests { + x := new(Int) + x.SetString(test.x, 0) + testBitset(t, x) + x = new(Int) + x.SetString(test.y, 0) + testBitset(t, x) + } + for i, test := range bitsetTests { + x := new(Int) + x.SetString(test.x, 0) + b := x.Bit(test.i) + if b != test.b { + + t.Errorf("#%d want %v got %v", i, test.b, b) + } + } +} + + +func BenchmarkBitset(b *testing.B) { + z := new(Int) + z.SetBit(z, 512, 1) + b.ResetTimer() + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + z.SetBit(z, i&512, 1) + } +} + + +func BenchmarkBitsetNeg(b *testing.B) { + z := NewInt(-1) + z.SetBit(z, 512, 0) + b.ResetTimer() + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + z.SetBit(z, i&512, 0) + } +} + + +func BenchmarkBitsetOrig(b *testing.B) { + z := new(Int) + altSetBit(z, z, 512, 1) + b.ResetTimer() + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + altSetBit(z, z, i&512, 1) + } +} + + +func BenchmarkBitsetNegOrig(b *testing.B) { + z := NewInt(-1) + altSetBit(z, z, 512, 0) + b.ResetTimer() + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + altSetBit(z, z, i&512, 0) + } +} + + func TestBitwise(t *testing.T) { x := new(Int) y := new(Int) @@ -1019,6 +1257,7 @@ var notTests = []struct { }, } + func TestNot(t *testing.T) { in := new(Int) out := new(Int) @@ -1047,6 +1286,7 @@ var modInverseTests = []struct { {"239487239847", "2410312426921032588552076022197566074856950548502459942654116941958108831682612228890093858261341614673227141477904012196503648957050582631942730706805009223062734745341073406696246014589361659774041027169249453200378729434170325843778659198143763193776859869524088940195577346119843545301547043747207749969763750084308926339295559968882457872412993810129130294592999947926365264059284647209730384947211681434464714438488520940127459844288859336526896320919633919"}, } + func TestModInverse(t *testing.T) { var element, prime Int one := NewInt(1) @@ -1063,6 +1303,7 @@ func TestModInverse(t *testing.T) { } +// used by TestIntGobEncoding and TestRatGobEncoding var gobEncodingTests = []string{ "0", "1", @@ -1073,7 +1314,7 @@ var gobEncodingTests = []string{ "298472983472983471903246121093472394872319615612417471234712061", } -func TestGobEncoding(t *testing.T) { +func TestIntGobEncoding(t *testing.T) { var medium bytes.Buffer enc := gob.NewEncoder(&medium) dec := gob.NewDecoder(&medium) @@ -1081,7 +1322,8 @@ func TestGobEncoding(t *testing.T) { for j := 0; j < 2; j++ { medium.Reset() // empty buffer for each test case (in case of failures) stest := test - if j == 0 { + if j != 0 { + // negative numbers stest = "-" + test } var tx Int diff --git a/src/pkg/big/nat.go b/src/pkg/big/nat.go index 4848d427b..734568e06 100755 --- a/src/pkg/big/nat.go +++ b/src/pkg/big/nat.go @@ -18,7 +18,12 @@ package big // These are the building blocks for the operations on signed integers // and rationals. -import "rand" +import ( + "io" + "os" + "rand" +) + // An unsigned integer x of the form // @@ -545,9 +550,14 @@ func (z nat) divLarge(u, uIn, v nat) (q, r nat) { u.clear() // D1. - shift := Word(leadingZeros(v[n-1])) - shlVW(v, v, shift) - u[len(uIn)] = shlVW(u[0:len(uIn)], uIn, shift) + shift := leadingZeros(v[n-1]) + if shift > 0 { + // do not modify v, it may be used by another goroutine simultaneously + v1 := make(nat, n) + shlVU(v1, v, shift) + v = v1 + } + u[len(uIn)] = shlVU(u[0:len(uIn)], uIn, shift) // D2. for j := m; j >= 0; j-- { @@ -586,8 +596,7 @@ func (z nat) divLarge(u, uIn, v nat) (q, r nat) { } q = q.norm() - shrVW(u, u, shift) - shrVW(v, v, shift) + shrVU(u, u, shift) r = u.norm() return q, r @@ -603,96 +612,252 @@ func (x nat) bitLen() int { } -func hexValue(ch byte) int { - var d byte +// MaxBase is the largest number base accepted for string conversions. +const MaxBase = 'z' - 'a' + 10 + 1 // = hexValue('z') + 1 + + +func hexValue(ch int) Word { + d := MaxBase + 1 // illegal base switch { case '0' <= ch && ch <= '9': d = ch - '0' - case 'a' <= ch && ch <= 'f': + case 'a' <= ch && ch <= 'z': d = ch - 'a' + 10 - case 'A' <= ch && ch <= 'F': + case 'A' <= ch && ch <= 'Z': d = ch - 'A' + 10 - default: - return -1 } - return int(d) + return Word(d) } -// scan returns the natural number corresponding to the -// longest possible prefix of s representing a natural number in a -// given conversion base, the actual conversion base used, and the -// prefix length. The syntax of natural numbers follows the syntax -// of unsigned integer literals in Go. +// scan sets z to the natural number corresponding to the longest possible prefix +// read from r representing an unsigned integer in a given conversion base. +// It returns z, the actual conversion base used, and an error, if any. In the +// error case, the value of z is undefined. The syntax follows the syntax of +// unsigned integer literals in Go. // -// If the base argument is 0, the string prefix determines the actual -// conversion base. A prefix of ``0x'' or ``0X'' selects base 16; the -// ``0'' prefix selects base 8, and a ``0b'' or ``0B'' prefix selects -// base 2. Otherwise the selected base is 10. +// The base argument must be 0 or a value from 2 through MaxBase. If the base +// is 0, the string prefix determines the actual conversion base. A prefix of +// ``0x'' or ``0X'' selects base 16; the ``0'' prefix selects base 8, and a +// ``0b'' or ``0B'' prefix selects base 2. Otherwise the selected base is 10. // -func (z nat) scan(s string, base int) (nat, int, int) { +func (z nat) scan(r io.RuneScanner, base int) (nat, int, os.Error) { + // reject illegal bases + if base < 0 || base == 1 || MaxBase < base { + return z, 0, os.ErrorString("illegal number base") + } + + // one char look-ahead + ch, _, err := r.ReadRune() + if err != nil { + return z, 0, err + } + // determine base if necessary - i, n := 0, len(s) + b := Word(base) if base == 0 { - base = 10 - if n > 0 && s[0] == '0' { - base, i = 8, 1 - if n > 1 { - switch s[1] { + b = 10 + if ch == '0' { + switch ch, _, err = r.ReadRune(); err { + case nil: + b = 8 + switch ch { case 'x', 'X': - base, i = 16, 2 + b = 16 case 'b', 'B': - base, i = 2, 2 + b = 2 } + if b == 2 || b == 16 { + if ch, _, err = r.ReadRune(); err != nil { + return z, 0, err + } + } + case os.EOF: + return z, 10, nil + default: + return z, 10, err } } } - // reject illegal bases or strings consisting only of prefix - if base < 2 || 16 < base || (base != 8 && i >= n) { - return z, 0, 0 - } - // convert string + // - group as many digits d as possible together into a "super-digit" dd with "super-base" bb + // - only when bb does not fit into a word anymore, do a full number mulAddWW using bb and dd z = z.make(0) - for ; i < n; i++ { - d := hexValue(s[i]) - if 0 <= d && d < base { - z = z.mulAddWW(z, Word(base), Word(d)) + bb := Word(1) + dd := Word(0) + for max := _M / b; ; { + d := hexValue(ch) + if d >= b { + r.UnreadRune() // ch does not belong to number anymore + break + } + + if bb <= max { + bb *= b + dd = dd*b + d } else { + // bb * b would overflow + z = z.mulAddWW(z, bb, dd) + bb = b + dd = d + } + + if ch, _, err = r.ReadRune(); err != nil { + if err != os.EOF { + return z, int(b), err + } break } } - return z.norm(), base, i + switch { + case bb > 1: + // there was at least one mantissa digit + z = z.mulAddWW(z, bb, dd) + case base == 0 && b == 8: + // there was only the octal prefix 0 (possibly followed by digits > 7); + // return base 10, not 8 + return z, 10, nil + case base != 0 || b != 8: + // there was neither a mantissa digit nor the octal prefix 0 + return z, int(b), os.ErrorString("syntax error scanning number") + } + + return z.norm(), int(b), nil } -// string converts x to a string for a given base, with 2 <= base <= 16. -// TODO(gri) in the style of the other routines, perhaps this should take -// a []byte buffer and return it -func (x nat) string(base int) string { - if base < 2 || 16 < base { - panic("illegal base") - } +// Character sets for string conversion. +const ( + lowercaseDigits = "0123456789abcdefghijklmnopqrstuvwxyz" + uppercaseDigits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" +) - if len(x) == 0 { - return "0" + +// decimalString returns a decimal representation of x. +// It calls x.string with the charset "0123456789". +func (x nat) decimalString() string { + return x.string(lowercaseDigits[0:10]) +} + + +// string converts x to a string using digits from a charset; a digit with +// value d is represented by charset[d]. The conversion base is determined +// by len(charset), which must be >= 2. +func (x nat) string(charset string) string { + b := Word(len(charset)) + + // special cases + switch { + case b < 2 || b > 256: + panic("illegal base") + case len(x) == 0: + return string(charset[0]) } // allocate buffer for conversion - i := x.bitLen()/log2(Word(base)) + 1 // +1: round up + i := x.bitLen()/log2(b) + 1 // +1: round up s := make([]byte, i) - // don't destroy x + // special case: power of two bases can avoid divisions completely + if b == b&-b { + // shift is base-b digit size in bits + shift := uint(trailingZeroBits(b)) // shift > 0 because b >= 2 + mask := Word(1)<<shift - 1 + w := x[0] + nbits := uint(_W) // number of unprocessed bits in w + + // convert less-significant words + for k := 1; k < len(x); k++ { + // convert full digits + for nbits >= shift { + i-- + s[i] = charset[w&mask] + w >>= shift + nbits -= shift + } + + // convert any partial leading digit and advance to next word + if nbits == 0 { + // no partial digit remaining, just advance + w = x[k] + nbits = _W + } else { + // partial digit in current (k-1) and next (k) word + w |= x[k] << nbits + i-- + s[i] = charset[w&mask] + + // advance + w = x[k] >> (shift - nbits) + nbits = _W - (shift - nbits) + } + } + + // convert digits of most-significant word (omit leading zeros) + for nbits >= 0 && w != 0 { + i-- + s[i] = charset[w&mask] + w >>= shift + nbits -= shift + } + + return string(s[i:]) + } + + // general case: extract groups of digits by multiprecision division + + // maximize ndigits where b**ndigits < 2^_W; bb (big base) is b**ndigits + bb := Word(1) + ndigits := 0 + for max := Word(_M / b); bb <= max; bb *= b { + ndigits++ + } + + // preserve x, create local copy for use in repeated divisions q := nat(nil).set(x) + var r Word // convert - for len(q) > 0 { - i-- - var r Word - q, r = q.divW(q, Word(base)) - s[i] = "0123456789abcdef"[r] + if b == 10 { // hard-coding for 10 here speeds this up by 1.25x + for len(q) > 0 { + // extract least significant, base bb "digit" + q, r = q.divW(q, bb) // N.B. >82% of time is here. Optimize divW + if len(q) == 0 { + // skip leading zeros in most-significant group of digits + for j := 0; j < ndigits && r != 0; j++ { + i-- + s[i] = charset[r%10] + r /= 10 + } + } else { + for j := 0; j < ndigits; j++ { + i-- + s[i] = charset[r%10] + r /= 10 + } + } + } + } else { + for len(q) > 0 { + // extract least significant group of digits + q, r = q.divW(q, bb) // N.B. >82% of time is here. Optimize divW + if len(q) == 0 { + // skip leading zeros in most-significant group of digits + for j := 0; j < ndigits && r != 0; j++ { + i-- + s[i] = charset[r%b] + r /= b + } + } else { + for j := 0; j < ndigits; j++ { + i-- + s[i] = charset[r%b] + r /= b + } + } + } } return string(s[i:]) @@ -715,13 +880,14 @@ var deBruijn64Lookup = []byte{ 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, } + // trailingZeroBits returns the number of consecutive zero bits on the right // side of the given Word. // See Knuth, volume 4, section 7.3.1 func trailingZeroBits(x Word) int { // x & -x leaves only the right-most bit set in the word. Let k be the // index of that bit. Since only a single bit is set, the value is two - // to the power of k. Multipling by a power of two is equivalent to + // to the power of k. Multiplying by a power of two is equivalent to // left shifting, in this case by k bits. The de Bruijn constant is // such that all six bit, consecutive substrings are distinct. // Therefore, if we have a left shifted version of this constant we can @@ -750,7 +916,7 @@ func (z nat) shl(x nat, s uint) nat { n := m + int(s/_W) z = z.make(n + 1) - z[n] = shlVW(z[n-m:n], x, Word(s%_W)) + z[n] = shlVU(z[n-m:n], x, s%_W) z[0 : n-m].clear() return z.norm() @@ -767,12 +933,49 @@ func (z nat) shr(x nat, s uint) nat { // n > 0 z = z.make(n) - shrVW(z, x[m-n:], Word(s%_W)) + shrVU(z, x[m-n:], s%_W) return z.norm() } +func (z nat) setBit(x nat, i uint, b uint) nat { + j := int(i / _W) + m := Word(1) << (i % _W) + n := len(x) + switch b { + case 0: + z = z.make(n) + copy(z, x) + if j >= n { + // no need to grow + return z + } + z[j] &^= m + return z.norm() + case 1: + if j >= n { + n = j + 1 + } + z = z.make(n) + copy(z, x) + z[j] |= m + // no need to normalize + return z + } + panic("set bit is not 0 or 1") +} + + +func (z nat) bit(i uint) uint { + j := int(i / _W) + if j >= len(z) { + return 0 + } + return uint(z[j] >> (i % _W) & 1) +} + + func (z nat) and(x, y nat) nat { m := len(x) n := len(y) @@ -849,7 +1052,9 @@ func (z nat) xor(x, y nat) nat { // greaterThan returns true iff (x1<<_W + x2) > (y1<<_W + y2) -func greaterThan(x1, x2, y1, y2 Word) bool { return x1 > y1 || x1 == y1 && x2 > y2 } +func greaterThan(x1, x2, y1, y2 Word) bool { + return x1 > y1 || x1 == y1 && x2 > y2 +} // modW returns x % d. @@ -861,25 +1066,26 @@ func (x nat) modW(d Word) (r Word) { } -// powersOfTwoDecompose finds q and k such that q * 1<<k = n and q is odd. -func (n nat) powersOfTwoDecompose() (q nat, k Word) { - if len(n) == 0 { - return n, 0 +// powersOfTwoDecompose finds q and k with x = q * 1<<k and q is odd, or q and k are 0. +func (x nat) powersOfTwoDecompose() (q nat, k int) { + if len(x) == 0 { + return x, 0 } - zeroWords := 0 - for n[zeroWords] == 0 { - zeroWords++ + // One of the words must be non-zero by definition, + // so this loop will terminate with i < len(x), and + // i is the number of 0 words. + i := 0 + for x[i] == 0 { + i++ } - // One of the words must be non-zero by invariant, therefore - // zeroWords < len(n). - x := trailingZeroBits(n[zeroWords]) + n := trailingZeroBits(x[i]) // x[i] != 0 - q = q.make(len(n) - zeroWords) - shrVW(q, n[zeroWords:], Word(x)) - q = q.norm() + q = make(nat, len(x)-i) + shrVU(q, x[i:], uint(n)) - k = Word(_W*zeroWords + x) + q = q.norm() + k = i*_W + n return } @@ -1050,7 +1256,7 @@ NextRandom: if y.cmp(natOne) == 0 || y.cmp(nm1) == 0 { continue } - for j := Word(1); j < k; j++ { + for j := 1; j < k; j++ { y = y.mul(y, y) quotient, y = quotient.div(y, y, n) if y.cmp(nm1) == 0 { diff --git a/src/pkg/big/nat_test.go b/src/pkg/big/nat_test.go index 0bcb94554..fd93592dd 100755 --- a/src/pkg/big/nat_test.go +++ b/src/pkg/big/nat_test.go @@ -4,7 +4,12 @@ package big -import "testing" +import ( + "fmt" + "os" + "strings" + "testing" +) var cmpTests = []struct { x, y nat @@ -133,7 +138,7 @@ var mulRangesN = []struct { func TestMulRangeN(t *testing.T) { for i, r := range mulRangesN { - prod := nat(nil).mulRange(r.a, r.b).string(10) + prod := nat(nil).mulRange(r.a, r.b).decimalString() if prod != r.prod { t.Errorf("#%d: got %s; want %s", i, prod, r.prod) } @@ -167,35 +172,402 @@ func BenchmarkMul(b *testing.B) { } -var tab = []struct { - x nat - b int - s string +func toString(x nat, charset string) string { + base := len(charset) + + // special cases + switch { + case base < 2: + panic("illegal base") + case len(x) == 0: + return string(charset[0]) + } + + // allocate buffer for conversion + i := x.bitLen()/log2(Word(base)) + 1 // +1: round up + s := make([]byte, i) + + // don't destroy x + q := nat(nil).set(x) + + // convert + for len(q) > 0 { + i-- + var r Word + q, r = q.divW(q, Word(base)) + s[i] = charset[r] + } + + return string(s[i:]) +} + + +var strTests = []struct { + x nat // nat value to be converted + c string // conversion charset + s string // expected result }{ - {nil, 10, "0"}, - {nat{1}, 10, "1"}, - {nat{10}, 10, "10"}, - {nat{1234567890}, 10, "1234567890"}, + {nil, "01", "0"}, + {nat{1}, "01", "1"}, + {nat{0xc5}, "01", "11000101"}, + {nat{03271}, lowercaseDigits[0:8], "3271"}, + {nat{10}, lowercaseDigits[0:10], "10"}, + {nat{1234567890}, uppercaseDigits[0:10], "1234567890"}, + {nat{0xdeadbeef}, lowercaseDigits[0:16], "deadbeef"}, + {nat{0xdeadbeef}, uppercaseDigits[0:16], "DEADBEEF"}, + {nat{0x229be7}, lowercaseDigits[0:17], "1a2b3c"}, + {nat{0x309663e6}, uppercaseDigits[0:32], "O9COV6"}, } func TestString(t *testing.T) { - for _, a := range tab { - s := a.x.string(a.b) + for _, a := range strTests { + s := a.x.string(a.c) if s != a.s { t.Errorf("string%+v\n\tgot s = %s; want %s", a, s, a.s) } - x, b, n := nat(nil).scan(a.s, a.b) + x, b, err := nat(nil).scan(strings.NewReader(a.s), len(a.c)) + if x.cmp(a.x) != 0 { + t.Errorf("scan%+v\n\tgot z = %v; want %v", a, x, a.x) + } + if b != len(a.c) { + t.Errorf("scan%+v\n\tgot b = %d; want %d", a, b, len(a.c)) + } + if err != nil { + t.Errorf("scan%+v\n\tgot error = %s", a, err) + } + } +} + + +var natScanTests = []struct { + s string // string to be scanned + base int // input base + x nat // expected nat + b int // expected base + ok bool // expected success + next int // next character (or 0, if at EOF) +}{ + // error: illegal base + {base: -1}, + {base: 1}, + {base: 37}, + + // error: no mantissa + {}, + {s: "?"}, + {base: 10}, + {base: 36}, + {s: "?", base: 10}, + {s: "0x"}, + {s: "345", base: 2}, + + // no errors + {"0", 0, nil, 10, true, 0}, + {"0", 10, nil, 10, true, 0}, + {"0", 36, nil, 36, true, 0}, + {"1", 0, nat{1}, 10, true, 0}, + {"1", 10, nat{1}, 10, true, 0}, + {"0 ", 0, nil, 10, true, ' '}, + {"08", 0, nil, 10, true, '8'}, + {"018", 0, nat{1}, 8, true, '8'}, + {"0b1", 0, nat{1}, 2, true, 0}, + {"0b11000101", 0, nat{0xc5}, 2, true, 0}, + {"03271", 0, nat{03271}, 8, true, 0}, + {"10ab", 0, nat{10}, 10, true, 'a'}, + {"1234567890", 0, nat{1234567890}, 10, true, 0}, + {"xyz", 36, nat{(33*36+34)*36 + 35}, 36, true, 0}, + {"xyz?", 36, nat{(33*36+34)*36 + 35}, 36, true, '?'}, + {"0x", 16, nil, 16, true, 'x'}, + {"0xdeadbeef", 0, nat{0xdeadbeef}, 16, true, 0}, + {"0XDEADBEEF", 0, nat{0xdeadbeef}, 16, true, 0}, +} + + +func TestScanBase(t *testing.T) { + for _, a := range natScanTests { + r := strings.NewReader(a.s) + x, b, err := nat(nil).scan(r, a.base) + if err == nil && !a.ok { + t.Errorf("scan%+v\n\texpected error", a) + } + if err != nil { + if a.ok { + t.Errorf("scan%+v\n\tgot error = %s", a, err) + } + continue + } if x.cmp(a.x) != 0 { t.Errorf("scan%+v\n\tgot z = %v; want %v", a, x, a.x) } if b != a.b { - t.Errorf("scan%+v\n\tgot b = %d; want %d", a, b, a.b) + t.Errorf("scan%+v\n\tgot b = %d; want %d", a, b, a.base) } - if n != len(a.s) { - t.Errorf("scan%+v\n\tgot n = %d; want %d", a, n, len(a.s)) + next, _, err := r.ReadRune() + if err == os.EOF { + next = 0 + err = nil } + if err == nil && next != a.next { + t.Errorf("scan%+v\n\tgot next = %q; want %q", a, next, a.next) + } + } +} + + +var pi = "3" + + "14159265358979323846264338327950288419716939937510582097494459230781640628620899862803482534211706798214808651" + + "32823066470938446095505822317253594081284811174502841027019385211055596446229489549303819644288109756659334461" + + "28475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920" + + "96282925409171536436789259036001133053054882046652138414695194151160943305727036575959195309218611738193261179" + + "31051185480744623799627495673518857527248912279381830119491298336733624406566430860213949463952247371907021798" + + "60943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901" + + "22495343014654958537105079227968925892354201995611212902196086403441815981362977477130996051870721134999999837" + + "29780499510597317328160963185950244594553469083026425223082533446850352619311881710100031378387528865875332083" + + "81420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909" + + "21642019893809525720106548586327886593615338182796823030195203530185296899577362259941389124972177528347913151" + + "55748572424541506959508295331168617278558890750983817546374649393192550604009277016711390098488240128583616035" + + "63707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104" + + "75216205696602405803815019351125338243003558764024749647326391419927260426992279678235478163600934172164121992" + + "45863150302861829745557067498385054945885869269956909272107975093029553211653449872027559602364806654991198818" + + "34797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548" + + "16136115735255213347574184946843852332390739414333454776241686251898356948556209921922218427255025425688767179" + + "04946016534668049886272327917860857843838279679766814541009538837863609506800642251252051173929848960841284886" + + "26945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645" + + "99581339047802759009946576407895126946839835259570982582262052248940772671947826848260147699090264013639443745" + + "53050682034962524517493996514314298091906592509372216964615157098583874105978859597729754989301617539284681382" + + "68683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244" + + "13654976278079771569143599770012961608944169486855584840635342207222582848864815845602850601684273945226746767" + + "88952521385225499546667278239864565961163548862305774564980355936345681743241125150760694794510965960940252288" + + "79710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821" + + "68299894872265880485756401427047755513237964145152374623436454285844479526586782105114135473573952311342716610" + + "21359695362314429524849371871101457654035902799344037420073105785390621983874478084784896833214457138687519435" + + "06430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675" + + "14269123974894090718649423196156794520809514655022523160388193014209376213785595663893778708303906979207734672" + + "21825625996615014215030680384477345492026054146659252014974428507325186660021324340881907104863317346496514539" + + "05796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007" + + "23055876317635942187312514712053292819182618612586732157919841484882916447060957527069572209175671167229109816" + + "90915280173506712748583222871835209353965725121083579151369882091444210067510334671103141267111369908658516398" + + "31501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064" + + "20467525907091548141654985946163718027098199430992448895757128289059232332609729971208443357326548938239119325" + + "97463667305836041428138830320382490375898524374417029132765618093773444030707469211201913020330380197621101100" + + "44929321516084244485963766983895228684783123552658213144957685726243344189303968642624341077322697802807318915" + + "44110104468232527162010526522721116603966655730925471105578537634668206531098965269186205647693125705863566201" + + "85581007293606598764861179104533488503461136576867532494416680396265797877185560845529654126654085306143444318" + + "58676975145661406800700237877659134401712749470420562230538994561314071127000407854733269939081454664645880797" + + "27082668306343285878569830523580893306575740679545716377525420211495576158140025012622859413021647155097925923" + + "09907965473761255176567513575178296664547791745011299614890304639947132962107340437518957359614589019389713111" + + "79042978285647503203198691514028708085990480109412147221317947647772622414254854540332157185306142288137585043" + + "06332175182979866223717215916077166925474873898665494945011465406284336639379003976926567214638530673609657120" + + "91807638327166416274888800786925602902284721040317211860820419000422966171196377921337575114959501566049631862" + + "94726547364252308177036751590673502350728354056704038674351362222477158915049530984448933309634087807693259939" + + "78054193414473774418426312986080998886874132604721569516239658645730216315981931951673538129741677294786724229" + + "24654366800980676928238280689964004824354037014163149658979409243237896907069779422362508221688957383798623001" + + "59377647165122893578601588161755782973523344604281512627203734314653197777416031990665541876397929334419521541" + + "34189948544473456738316249934191318148092777710386387734317720754565453220777092120190516609628049092636019759" + + "88281613323166636528619326686336062735676303544776280350450777235547105859548702790814356240145171806246436267" + + "94561275318134078330336254232783944975382437205835311477119926063813346776879695970309833913077109870408591337" + + +// Test case for BenchmarkScanPi. +func TestScanPi(t *testing.T) { + var x nat + z, _, err := x.scan(strings.NewReader(pi), 10) + if err != nil { + t.Errorf("scanning pi: %s", err) + } + if s := z.decimalString(); s != pi { + t.Errorf("scanning pi: got %s", s) + } +} + + +func BenchmarkScanPi(b *testing.B) { + for i := 0; i < b.N; i++ { + var x nat + x.scan(strings.NewReader(pi), 10) + } +} + + +const ( + // 314**271 + // base 2: 2249 digits + // base 8: 751 digits + // base 10: 678 digits + // base 16: 563 digits + shortBase = 314 + shortExponent = 271 + + // 3141**2178 + // base 2: 31577 digits + // base 8: 10527 digits + // base 10: 9507 digits + // base 16: 7895 digits + mediumBase = 3141 + mediumExponent = 2718 + + // 3141**2178 + // base 2: 406078 digits + // base 8: 135360 digits + // base 10: 122243 digits + // base 16: 101521 digits + longBase = 31415 + longExponent = 27182 +) + + +func BenchmarkScanShort2(b *testing.B) { + ScanHelper(b, 2, shortBase, shortExponent) +} + + +func BenchmarkScanShort8(b *testing.B) { + ScanHelper(b, 8, shortBase, shortExponent) +} + + +func BenchmarkScanSort10(b *testing.B) { + ScanHelper(b, 10, shortBase, shortExponent) +} + + +func BenchmarkScanShort16(b *testing.B) { + ScanHelper(b, 16, shortBase, shortExponent) +} + + +func BenchmarkScanMedium2(b *testing.B) { + ScanHelper(b, 2, mediumBase, mediumExponent) +} + + +func BenchmarkScanMedium8(b *testing.B) { + ScanHelper(b, 8, mediumBase, mediumExponent) +} + + +func BenchmarkScanMedium10(b *testing.B) { + ScanHelper(b, 10, mediumBase, mediumExponent) +} + + +func BenchmarkScanMedium16(b *testing.B) { + ScanHelper(b, 16, mediumBase, mediumExponent) +} + + +func BenchmarkScanLong2(b *testing.B) { + ScanHelper(b, 2, longBase, longExponent) +} + + +func BenchmarkScanLong8(b *testing.B) { + ScanHelper(b, 8, longBase, longExponent) +} + + +func BenchmarkScanLong10(b *testing.B) { + ScanHelper(b, 10, longBase, longExponent) +} + + +func BenchmarkScanLong16(b *testing.B) { + ScanHelper(b, 16, longBase, longExponent) +} + + +func ScanHelper(b *testing.B, base int, xv, yv Word) { + b.StopTimer() + var x, y, z nat + x = x.setWord(xv) + y = y.setWord(yv) + z = z.expNN(x, y, nil) + + var s string + s = z.string(lowercaseDigits[0:base]) + if t := toString(z, lowercaseDigits[0:base]); t != s { + panic(fmt.Sprintf("scanning: got %s; want %s", s, t)) + } + b.StartTimer() + + for i := 0; i < b.N; i++ { + x.scan(strings.NewReader(s), base) + } +} + + +func BenchmarkStringShort2(b *testing.B) { + StringHelper(b, 2, shortBase, shortExponent) +} + + +func BenchmarkStringShort8(b *testing.B) { + StringHelper(b, 8, shortBase, shortExponent) +} + + +func BenchmarkStringShort10(b *testing.B) { + StringHelper(b, 10, shortBase, shortExponent) +} + + +func BenchmarkStringShort16(b *testing.B) { + StringHelper(b, 16, shortBase, shortExponent) +} + + +func BenchmarkStringMedium2(b *testing.B) { + StringHelper(b, 2, mediumBase, mediumExponent) +} + + +func BenchmarkStringMedium8(b *testing.B) { + StringHelper(b, 8, mediumBase, mediumExponent) +} + + +func BenchmarkStringMedium10(b *testing.B) { + StringHelper(b, 10, mediumBase, mediumExponent) +} + + +func BenchmarkStringMedium16(b *testing.B) { + StringHelper(b, 16, mediumBase, mediumExponent) +} + + +func BenchmarkStringLong2(b *testing.B) { + StringHelper(b, 2, longBase, longExponent) +} + + +func BenchmarkStringLong8(b *testing.B) { + StringHelper(b, 8, longBase, longExponent) +} + + +func BenchmarkStringLong10(b *testing.B) { + StringHelper(b, 10, longBase, longExponent) +} + + +func BenchmarkStringLong16(b *testing.B) { + StringHelper(b, 16, longBase, longExponent) +} + + +func StringHelper(b *testing.B, base int, xv, yv Word) { + b.StopTimer() + var x, y, z nat + x = x.setWord(xv) + y = y.setWord(yv) + z = z.expNN(x, y, nil) + b.StartTimer() + + for i := 0; i < b.N; i++ { + z.string(lowercaseDigits[0:base]) } } @@ -319,7 +691,6 @@ func TestTrailingZeroBits(t *testing.T) { } } - var expNNTests = []struct { x, y, m string out string @@ -340,14 +711,14 @@ var expNNTests = []struct { func TestExpNN(t *testing.T) { for i, test := range expNNTests { - x, _, _ := nat(nil).scan(test.x, 0) - y, _, _ := nat(nil).scan(test.y, 0) - out, _, _ := nat(nil).scan(test.out, 0) + x, _, _ := nat(nil).scan(strings.NewReader(test.x), 0) + y, _, _ := nat(nil).scan(strings.NewReader(test.y), 0) + out, _, _ := nat(nil).scan(strings.NewReader(test.out), 0) var m nat if len(test.m) > 0 { - m, _, _ = nat(nil).scan(test.m, 0) + m, _, _ = nat(nil).scan(strings.NewReader(test.m), 0) } z := nat(nil).expNN(x, y, m) diff --git a/src/pkg/big/rat.go b/src/pkg/big/rat.go index e70673a1c..1fbf8c459 100644 --- a/src/pkg/big/rat.go +++ b/src/pkg/big/rat.go @@ -6,7 +6,12 @@ package big -import "strings" +import ( + "encoding/binary" + "fmt" + "os" + "strings" +) // A Rat represents a quotient a/b of arbitrary precision. The zero value for // a Rat, 0/0, is not a legal Rat. @@ -84,7 +89,7 @@ func (z *Rat) Num() *Int { } -// Demom returns the denominator of z; it is always > 0. +// Denom returns the denominator of z; it is always > 0. // The result is a reference to z's denominator; it // may change if a new value is assigned to z. func (z *Rat) Denom() *Int { @@ -209,6 +214,28 @@ func (z *Rat) Set(x *Rat) *Rat { } +func ratTok(ch int) bool { + return strings.IndexRune("+-/0123456789.eE", ch) >= 0 +} + + +// Scan is a support routine for fmt.Scanner. It accepts the formats +// 'e', 'E', 'f', 'F', 'g', 'G', and 'v'. All formats are equivalent. +func (z *Rat) Scan(s fmt.ScanState, ch int) os.Error { + tok, err := s.Token(true, ratTok) + if err != nil { + return err + } + if strings.IndexRune("efgEFGv", ch) < 0 { + return os.ErrorString("Rat.Scan: invalid verb") + } + if _, ok := z.SetString(string(tok)); !ok { + return os.ErrorString("Rat.Scan: invalid syntax") + } + return nil +} + + // SetString sets z to the value of s and returns z and a boolean indicating // success. s can be given as a fraction "a/b" or as a floating-point number // optionally followed by an exponent. If the operation failed, the value of z @@ -225,8 +252,8 @@ func (z *Rat) SetString(s string) (*Rat, bool) { return z, false } s = s[sep+1:] - var n int - if z.b, _, n = z.b.scan(s, 10); n != len(s) { + var err os.Error + if z.b, _, err = z.b.scan(strings.NewReader(s), 10); err != nil { return z, false } return z.norm(), true @@ -270,7 +297,7 @@ func (z *Rat) SetString(s string) (*Rat, bool) { // String returns a string representation of z in the form "a/b" (even if b == 1). func (z *Rat) String() string { - return z.a.String() + "/" + z.b.string(10) + return z.a.String() + "/" + z.b.decimalString() } @@ -288,7 +315,11 @@ func (z *Rat) RatString() string { // digits of precision after the decimal point and the last digit rounded. func (z *Rat) FloatString(prec int) string { if z.IsInt() { - return z.a.String() + s := z.a.String() + if prec > 0 { + s += "." + strings.Repeat("0", prec) + } + return s } q, r := nat{}.div(nat{}, z.a.abs, z.b) @@ -311,16 +342,58 @@ func (z *Rat) FloatString(prec int) string { } } - s := q.string(10) + s := q.decimalString() if z.a.neg { s = "-" + s } if prec > 0 { - rs := r.string(10) + rs := r.decimalString() leadingZeros := prec - len(rs) s += "." + strings.Repeat("0", leadingZeros) + rs } return s } + + +// Gob codec version. Permits backward-compatible changes to the encoding. +const ratGobVersion byte = 1 + +// GobEncode implements the gob.GobEncoder interface. +func (z *Rat) GobEncode() ([]byte, os.Error) { + buf := make([]byte, 1+4+(len(z.a.abs)+len(z.b))*_S) // extra bytes for version and sign bit (1), and numerator length (4) + i := z.b.bytes(buf) + j := z.a.abs.bytes(buf[0:i]) + n := i - j + if int(uint32(n)) != n { + // this should never happen + return nil, os.ErrorString("Rat.GobEncode: numerator too large") + } + binary.BigEndian.PutUint32(buf[j-4:j], uint32(n)) + j -= 1 + 4 + b := ratGobVersion << 1 // make space for sign bit + if z.a.neg { + b |= 1 + } + buf[j] = b + return buf[j:], nil +} + + +// GobDecode implements the gob.GobDecoder interface. +func (z *Rat) GobDecode(buf []byte) os.Error { + if len(buf) == 0 { + return os.ErrorString("Rat.GobDecode: no data") + } + b := buf[0] + if b>>1 != ratGobVersion { + return os.ErrorString(fmt.Sprintf("Rat.GobDecode: encoding version %d not supported", b>>1)) + } + const j = 1 + 4 + i := j + binary.BigEndian.Uint32(buf[j-4:j]) + z.a.neg = b&1 != 0 + z.a.abs = z.a.abs.setBytes(buf[j:i]) + z.b = z.b.setBytes(buf[i:]) + return nil +} diff --git a/src/pkg/big/rat_test.go b/src/pkg/big/rat_test.go index 8f42949b0..e64505ea3 100644 --- a/src/pkg/big/rat_test.go +++ b/src/pkg/big/rat_test.go @@ -4,7 +4,12 @@ package big -import "testing" +import ( + "bytes" + "fmt" + "gob" + "testing" +) var setStringTests = []struct { @@ -53,18 +58,42 @@ func TestRatSetString(t *testing.T) { } +func TestRatScan(t *testing.T) { + var buf bytes.Buffer + for i, test := range setStringTests { + x := new(Rat) + buf.Reset() + buf.WriteString(test.in) + + _, err := fmt.Fscanf(&buf, "%v", x) + if err == nil != test.ok { + if test.ok { + t.Errorf("#%d error: %s", i, err.String()) + } else { + t.Errorf("#%d expected error", i) + } + continue + } + if err == nil && x.RatString() != test.out { + t.Errorf("#%d got %s want %s", i, x.RatString(), test.out) + } + } +} + + var floatStringTests = []struct { in string prec int out string }{ {"0", 0, "0"}, - {"0", 4, "0"}, + {"0", 4, "0.0000"}, {"1", 0, "1"}, - {"1", 2, "1"}, + {"1", 2, "1.00"}, {"-1", 0, "-1"}, {".25", 2, "0.25"}, {".25", 1, "0.3"}, + {".25", 3, "0.250"}, {"-1/3", 3, "-0.333"}, {"-2/3", 4, "-0.6667"}, {"0.96", 1, "1.0"}, @@ -280,3 +309,36 @@ func TestRatSetFrac64Rat(t *testing.T) { } } } + + +func TestRatGobEncoding(t *testing.T) { + var medium bytes.Buffer + enc := gob.NewEncoder(&medium) + dec := gob.NewDecoder(&medium) + for i, test := range gobEncodingTests { + for j := 0; j < 4; j++ { + medium.Reset() // empty buffer for each test case (in case of failures) + stest := test + if j&1 != 0 { + // negative numbers + stest = "-" + test + } + if j%2 != 0 { + // fractions + stest = stest + "." + test + } + var tx Rat + tx.SetString(stest) + if err := enc.Encode(&tx); err != nil { + t.Errorf("#%d%c: encoding failed: %s", i, 'a'+j, err) + } + var rx Rat + if err := dec.Decode(&rx); err != nil { + t.Errorf("#%d%c: decoding failed: %s", i, 'a'+j, err) + } + if rx.Cmp(&tx) != 0 { + t.Errorf("#%d%c: transmission failed: got %s want %s", i, 'a'+j, &rx, &tx) + } + } + } +} diff --git a/src/pkg/compress/bzip2/bzip2.go b/src/pkg/compress/bzip2/bzip2.go index 9e97edec1..8b4572306 100644 --- a/src/pkg/compress/bzip2/bzip2.go +++ b/src/pkg/compress/bzip2/bzip2.go @@ -284,7 +284,7 @@ func (bz2 *reader) readBlock() (err os.Error) { repeat := 0 repeat_power := 0 - // The `C' array (used by the inverse BWT) needs to be zero initialised. + // The `C' array (used by the inverse BWT) needs to be zero initialized. for i := range bz2.c { bz2.c[i] = 0 } @@ -330,7 +330,7 @@ func (bz2 *reader) readBlock() (err os.Error) { if int(v) == numSymbols-1 { // This is the EOF symbol. Because it's always at the - // end of the move-to-front list, and nevers gets moved + // end of the move-to-front list, and never gets moved // to the front, it has this unique value. break } diff --git a/src/pkg/compress/bzip2/huffman.go b/src/pkg/compress/bzip2/huffman.go index 732bc4a21..dc05739c7 100644 --- a/src/pkg/compress/bzip2/huffman.go +++ b/src/pkg/compress/bzip2/huffman.go @@ -68,7 +68,7 @@ func newHuffmanTree(lengths []uint8) (huffmanTree, os.Error) { // each symbol (consider reflecting a tree down the middle, for // example). Since the code length assignments determine the // efficiency of the tree, each of these trees is equally good. In - // order to minimise the amount of information needed to build a tree + // order to minimize the amount of information needed to build a tree // bzip2 uses a canonical tree so that it can be reconstructed given // only the code length assignments. diff --git a/src/pkg/compress/flate/deflate.go b/src/pkg/compress/flate/deflate.go index a02a5e8d9..b1cee0b2f 100644 --- a/src/pkg/compress/flate/deflate.go +++ b/src/pkg/compress/flate/deflate.go @@ -11,16 +11,18 @@ import ( ) const ( - NoCompression = 0 - BestSpeed = 1 - fastCompression = 3 - BestCompression = 9 - DefaultCompression = -1 - logMaxOffsetSize = 15 // Standard DEFLATE - wideLogMaxOffsetSize = 22 // Wide DEFLATE - minMatchLength = 3 // The smallest match that the compressor looks for - maxMatchLength = 258 // The longest match for the compressor - minOffsetSize = 1 // The shortest offset that makes any sence + NoCompression = 0 + BestSpeed = 1 + fastCompression = 3 + BestCompression = 9 + DefaultCompression = -1 + logWindowSize = 15 + windowSize = 1 << logWindowSize + windowMask = windowSize - 1 + logMaxOffsetSize = 15 // Standard DEFLATE + minMatchLength = 3 // The smallest match that the compressor looks for + maxMatchLength = 258 // The longest match for the compressor + minOffsetSize = 1 // The shortest offset that makes any sence // The maximum number of tokens we put into a single flat block, just too // stop things from getting too large. @@ -32,22 +34,6 @@ const ( hashShift = (hashBits + minMatchLength - 1) / minMatchLength ) -type syncPipeReader struct { - *io.PipeReader - closeChan chan bool -} - -func (sr *syncPipeReader) CloseWithError(err os.Error) os.Error { - retErr := sr.PipeReader.CloseWithError(err) - sr.closeChan <- true // finish writer close - return retErr -} - -type syncPipeWriter struct { - *io.PipeWriter - closeChan chan bool -} - type compressionLevel struct { good, lazy, nice, chain, fastSkipHashing int } @@ -68,105 +54,73 @@ var levels = []compressionLevel{ {32, 258, 258, 4096, math.MaxInt32}, } -func (sw *syncPipeWriter) Close() os.Error { - err := sw.PipeWriter.Close() - <-sw.closeChan // wait for reader close - return err -} - -func syncPipe() (*syncPipeReader, *syncPipeWriter) { - r, w := io.Pipe() - sr := &syncPipeReader{r, make(chan bool, 1)} - sw := &syncPipeWriter{w, sr.closeChan} - return sr, sw -} - type compressor struct { - level int - logWindowSize uint - w *huffmanBitWriter - r io.Reader - // (1 << logWindowSize) - 1. - windowMask int + compressionLevel - eof bool // has eof been reached on input? - sync bool // writer wants to flush - syncChan chan os.Error + w *huffmanBitWriter - // hashHead[hashValue] contains the largest inputIndex with the specified hash value - hashHead []int + // compression algorithm + fill func(*compressor, []byte) int // copy data to window + step func(*compressor) // process window + sync bool // requesting flush + // Input hash chains + // hashHead[hashValue] contains the largest inputIndex with the specified hash value // If hashHead[hashValue] is within the current window, then // hashPrev[hashHead[hashValue] & windowMask] contains the previous index // with the same hash value. - hashPrev []int - - // If we find a match of length >= niceMatch, then we don't bother searching - // any further. - niceMatch int - - // If we find a match of length >= goodMatch, we only do a half-hearted - // effort at doing lazy matching starting at the next character - goodMatch int - - // The maximum number of chains we look at when finding a match - maxChainLength int - - // The sliding window we use for matching - window []byte - - // The index just past the last valid character - windowEnd int - - // index in "window" at which current block starts - blockStart int -} - -func (d *compressor) flush() os.Error { - d.w.flush() - return d.w.err + chainHead int + hashHead []int + hashPrev []int + + // input window: unprocessed data is window[index:windowEnd] + index int + window []byte + windowEnd int + blockStart int // window index where current tokens start + byteAvailable bool // if true, still need to process window[index-1]. + + // queued output tokens: tokens[:ti] + tokens []token + ti int + + // deflate state + length int + offset int + hash int + maxInsertIndex int + err os.Error } -func (d *compressor) fillWindow(index int) (int, os.Error) { - if d.sync { - return index, nil - } - wSize := d.windowMask + 1 - if index >= wSize+wSize-(minMatchLength+maxMatchLength) { - // shift the window by wSize - copy(d.window, d.window[wSize:2*wSize]) - index -= wSize - d.windowEnd -= wSize - if d.blockStart >= wSize { - d.blockStart -= wSize +func (d *compressor) fillDeflate(b []byte) int { + if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) { + // shift the window by windowSize + copy(d.window, d.window[windowSize:2*windowSize]) + d.index -= windowSize + d.windowEnd -= windowSize + if d.blockStart >= windowSize { + d.blockStart -= windowSize } else { d.blockStart = math.MaxInt32 } for i, h := range d.hashHead { - v := h - wSize + v := h - windowSize if v < -1 { v = -1 } d.hashHead[i] = v } for i, h := range d.hashPrev { - v := -h - wSize + v := -h - windowSize if v < -1 { v = -1 } d.hashPrev[i] = v } } - count, err := d.r.Read(d.window[d.windowEnd:]) - d.windowEnd += count - if count == 0 && err == nil { - d.sync = true - } - if err == os.EOF { - d.eof = true - err = nil - } - return index, err + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n } func (d *compressor) writeBlock(tokens []token, index int, eof bool) os.Error { @@ -194,21 +148,21 @@ func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead // We quit when we get a match that's at least nice long nice := len(win) - pos - if d.niceMatch < nice { - nice = d.niceMatch + if d.nice < nice { + nice = d.nice } // If we've got a match that's good enough, only look in 1/4 the chain. - tries := d.maxChainLength + tries := d.chain length = prevLength - if length >= d.goodMatch { + if length >= d.good { tries >>= 2 } w0 := win[pos] w1 := win[pos+1] wEnd := win[pos+length] - minIndex := pos - (d.windowMask + 1) + minIndex := pos - windowSize for i := prevHead; tries > 0; tries-- { if w0 == win[i] && w1 == win[i+1] && wEnd == win[i+length] { @@ -233,7 +187,7 @@ func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead // hashPrev[i & windowMask] has already been overwritten, so stop now. break } - if i = d.hashPrev[i&d.windowMask]; i < minIndex || i < 0 { + if i = d.hashPrev[i&windowMask]; i < minIndex || i < 0 { break } } @@ -248,234 +202,224 @@ func (d *compressor) writeStoredBlock(buf []byte) os.Error { return d.w.err } -func (d *compressor) storedDeflate() os.Error { - buf := make([]byte, maxStoreBlockSize) - for { - n, err := d.r.Read(buf) - if n == 0 && err == nil { - d.sync = true - } - if n > 0 || d.sync { - if err := d.writeStoredBlock(buf[0:n]); err != nil { - return err - } - if d.sync { - d.syncChan <- nil - d.sync = false - } - } - if err != nil { - if err == os.EOF { - break - } - return err - } - } - return nil -} - -func (d *compressor) doDeflate() (err os.Error) { - // init - d.windowMask = 1<<d.logWindowSize - 1 +func (d *compressor) initDeflate() { d.hashHead = make([]int, hashSize) - d.hashPrev = make([]int, 1<<d.logWindowSize) - d.window = make([]byte, 2<<d.logWindowSize) + d.hashPrev = make([]int, windowSize) + d.window = make([]byte, 2*windowSize) fillInts(d.hashHead, -1) - tokens := make([]token, maxFlateBlockTokens, maxFlateBlockTokens+1) - l := levels[d.level] - d.goodMatch = l.good - d.niceMatch = l.nice - d.maxChainLength = l.chain - lazyMatch := l.lazy - length := minMatchLength - 1 - offset := 0 - byteAvailable := false - isFastDeflate := l.fastSkipHashing != 0 - index := 0 - // run - if index, err = d.fillWindow(index); err != nil { + d.tokens = make([]token, maxFlateBlockTokens, maxFlateBlockTokens+1) + d.length = minMatchLength - 1 + d.offset = 0 + d.byteAvailable = false + d.index = 0 + d.ti = 0 + d.hash = 0 + d.chainHead = -1 +} + +func (d *compressor) deflate() { + if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { return } - maxOffset := d.windowMask + 1 // (1 << logWindowSize); - // only need to change when you refill the window - windowEnd := d.windowEnd - maxInsertIndex := windowEnd - (minMatchLength - 1) - ti := 0 - - hash := int(0) - if index < maxInsertIndex { - hash = int(d.window[index])<<hashShift + int(d.window[index+1]) + + d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) + if d.index < d.maxInsertIndex { + d.hash = int(d.window[d.index])<<hashShift + int(d.window[d.index+1]) } - chainHead := -1 + Loop: for { - if index > windowEnd { + if d.index > d.windowEnd { panic("index > windowEnd") } - lookahead := windowEnd - index + lookahead := d.windowEnd - d.index if lookahead < minMatchLength+maxMatchLength { - if index, err = d.fillWindow(index); err != nil { - return + if !d.sync { + break Loop } - windowEnd = d.windowEnd - if index > windowEnd { + if d.index > d.windowEnd { panic("index > windowEnd") } - maxInsertIndex = windowEnd - (minMatchLength - 1) - lookahead = windowEnd - index if lookahead == 0 { // Flush current output block if any. - if byteAvailable { + if d.byteAvailable { // There is still one pending token that needs to be flushed - tokens[ti] = literalToken(uint32(d.window[index-1]) & 0xFF) - ti++ - byteAvailable = false + d.tokens[d.ti] = literalToken(uint32(d.window[d.index-1])) + d.ti++ + d.byteAvailable = false } - if ti > 0 { - if err = d.writeBlock(tokens[0:ti], index, false); err != nil { + if d.ti > 0 { + if d.err = d.writeBlock(d.tokens[0:d.ti], d.index, false); d.err != nil { return } - ti = 0 - } - if d.sync { - d.w.writeStoredHeader(0, false) - d.w.flush() - d.syncChan <- d.w.err - d.sync = false - } - - // If this was only a sync (not at EOF) keep going. - if !d.eof { - continue + d.ti = 0 } break Loop } } - if index < maxInsertIndex { + if d.index < d.maxInsertIndex { // Update the hash - hash = (hash<<hashShift + int(d.window[index+2])) & hashMask - chainHead = d.hashHead[hash] - d.hashPrev[index&d.windowMask] = chainHead - d.hashHead[hash] = index + d.hash = (d.hash<<hashShift + int(d.window[d.index+2])) & hashMask + d.chainHead = d.hashHead[d.hash] + d.hashPrev[d.index&windowMask] = d.chainHead + d.hashHead[d.hash] = d.index } - prevLength := length - prevOffset := offset - length = minMatchLength - 1 - offset = 0 - minIndex := index - maxOffset + prevLength := d.length + prevOffset := d.offset + d.length = minMatchLength - 1 + d.offset = 0 + minIndex := d.index - windowSize if minIndex < 0 { minIndex = 0 } - if chainHead >= minIndex && - (isFastDeflate && lookahead > minMatchLength-1 || - !isFastDeflate && lookahead > prevLength && prevLength < lazyMatch) { - if newLength, newOffset, ok := d.findMatch(index, chainHead, minMatchLength-1, lookahead); ok { - length = newLength - offset = newOffset + if d.chainHead >= minIndex && + (d.fastSkipHashing != 0 && lookahead > minMatchLength-1 || + d.fastSkipHashing == 0 && lookahead > prevLength && prevLength < d.lazy) { + if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead, minMatchLength-1, lookahead); ok { + d.length = newLength + d.offset = newOffset } } - if isFastDeflate && length >= minMatchLength || - !isFastDeflate && prevLength >= minMatchLength && length <= prevLength { + if d.fastSkipHashing != 0 && d.length >= minMatchLength || + d.fastSkipHashing == 0 && prevLength >= minMatchLength && d.length <= prevLength { // There was a match at the previous step, and the current match is // not better. Output the previous match. - if isFastDeflate { - tokens[ti] = matchToken(uint32(length-minMatchLength), uint32(offset-minOffsetSize)) + if d.fastSkipHashing != 0 { + d.tokens[d.ti] = matchToken(uint32(d.length-minMatchLength), uint32(d.offset-minOffsetSize)) } else { - tokens[ti] = matchToken(uint32(prevLength-minMatchLength), uint32(prevOffset-minOffsetSize)) + d.tokens[d.ti] = matchToken(uint32(prevLength-minMatchLength), uint32(prevOffset-minOffsetSize)) } - ti++ + d.ti++ // Insert in the hash table all strings up to the end of the match. // index and index-1 are already inserted. If there is not enough // lookahead, the last two strings are not inserted into the hash // table. - if length <= l.fastSkipHashing { + if d.length <= d.fastSkipHashing { var newIndex int - if isFastDeflate { - newIndex = index + length + if d.fastSkipHashing != 0 { + newIndex = d.index + d.length } else { newIndex = prevLength - 1 } - for index++; index < newIndex; index++ { - if index < maxInsertIndex { - hash = (hash<<hashShift + int(d.window[index+2])) & hashMask + for d.index++; d.index < newIndex; d.index++ { + if d.index < d.maxInsertIndex { + d.hash = (d.hash<<hashShift + int(d.window[d.index+2])) & hashMask // Get previous value with the same hash. // Our chain should point to the previous value. - d.hashPrev[index&d.windowMask] = d.hashHead[hash] + d.hashPrev[d.index&windowMask] = d.hashHead[d.hash] // Set the head of the hash chain to us. - d.hashHead[hash] = index + d.hashHead[d.hash] = d.index } } - if !isFastDeflate { - byteAvailable = false - length = minMatchLength - 1 + if d.fastSkipHashing == 0 { + d.byteAvailable = false + d.length = minMatchLength - 1 } } else { // For matches this long, we don't bother inserting each individual // item into the table. - index += length - hash = (int(d.window[index])<<hashShift + int(d.window[index+1])) + d.index += d.length + d.hash = (int(d.window[d.index])<<hashShift + int(d.window[d.index+1])) } - if ti == maxFlateBlockTokens { + if d.ti == maxFlateBlockTokens { // The block includes the current character - if err = d.writeBlock(tokens, index, false); err != nil { + if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { return } - ti = 0 + d.ti = 0 } } else { - if isFastDeflate || byteAvailable { - i := index - 1 - if isFastDeflate { - i = index + if d.fastSkipHashing != 0 || d.byteAvailable { + i := d.index - 1 + if d.fastSkipHashing != 0 { + i = d.index } - tokens[ti] = literalToken(uint32(d.window[i]) & 0xFF) - ti++ - if ti == maxFlateBlockTokens { - if err = d.writeBlock(tokens, i+1, false); err != nil { + d.tokens[d.ti] = literalToken(uint32(d.window[i])) + d.ti++ + if d.ti == maxFlateBlockTokens { + if d.err = d.writeBlock(d.tokens, i+1, false); d.err != nil { return } - ti = 0 + d.ti = 0 } } - index++ - if !isFastDeflate { - byteAvailable = true + d.index++ + if d.fastSkipHashing == 0 { + d.byteAvailable = true } } } - return } -func (d *compressor) compress(r io.Reader, w io.Writer, level int, logWindowSize uint) (err os.Error) { - d.r = r +func (d *compressor) fillStore(b []byte) int { + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n +} + +func (d *compressor) store() { + if d.windowEnd > 0 { + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + } + d.windowEnd = 0 +} + +func (d *compressor) write(b []byte) (n int, err os.Error) { + n = len(b) + b = b[d.fill(d, b):] + for len(b) > 0 { + d.step(d) + b = b[d.fill(d, b):] + } + return n, d.err +} + +func (d *compressor) syncFlush() os.Error { + d.sync = true + d.step(d) + if d.err == nil { + d.w.writeStoredHeader(0, false) + d.w.flush() + d.err = d.w.err + } + d.sync = false + return d.err +} + +func (d *compressor) init(w io.Writer, level int) (err os.Error) { d.w = newHuffmanBitWriter(w) - d.level = level - d.logWindowSize = logWindowSize switch { case level == NoCompression: - err = d.storedDeflate() + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillStore + d.step = (*compressor).store case level == DefaultCompression: - d.level = 6 + level = 6 fallthrough case 1 <= level && level <= 9: - err = d.doDeflate() + d.compressionLevel = levels[level] + d.initDeflate() + d.fill = (*compressor).fillDeflate + d.step = (*compressor).deflate default: return WrongValueError{"level", 0, 9, int32(level)} } + return nil +} - if d.sync { - d.syncChan <- err - d.sync = false - } - if err != nil { - return err +func (d *compressor) close() os.Error { + d.sync = true + d.step(d) + if d.err != nil { + return d.err } if d.w.writeStoredHeader(0, true); d.w.err != nil { return d.w.err } - return d.flush() + d.w.flush() + return d.w.err } // NewWriter returns a new Writer compressing @@ -486,14 +430,9 @@ func (d *compressor) compress(r io.Reader, w io.Writer, level int, logWindowSize // compression; it only adds the necessary DEFLATE framing. func NewWriter(w io.Writer, level int) *Writer { const logWindowSize = logMaxOffsetSize - var d compressor - d.syncChan = make(chan os.Error, 1) - pr, pw := syncPipe() - go func() { - err := d.compress(pr, w, level, logWindowSize) - pr.CloseWithError(err) - }() - return &Writer{pw, &d} + var dw Writer + dw.d.init(w, level) + return &dw } // NewWriterDict is like NewWriter but initializes the new @@ -526,18 +465,13 @@ func (w *dictWriter) Write(b []byte) (n int, err os.Error) { // A Writer takes data written to it and writes the compressed // form of that data to an underlying writer (see NewWriter). type Writer struct { - w *syncPipeWriter - d *compressor + d compressor } // Write writes data to w, which will eventually write the // compressed form of data to its underlying writer. func (w *Writer) Write(data []byte) (n int, err os.Error) { - if len(data) == 0 { - // no point, and nil interferes with sync - return - } - return w.w.Write(data) + return w.d.write(data) } // Flush flushes any pending compressed data to the underlying writer. @@ -550,18 +484,10 @@ func (w *Writer) Write(data []byte) (n int, err os.Error) { func (w *Writer) Flush() os.Error { // For more about flushing: // http://www.bolet.org/~pornin/deflate-flush.html - if w.d.sync { - panic("compress/flate: double Flush") - } - _, err := w.w.Write(nil) - err1 := <-w.d.syncChan - if err == nil { - err = err1 - } - return err + return w.d.syncFlush() } // Close flushes and closes the writer. func (w *Writer) Close() os.Error { - return w.w.Close() + return w.d.close() } diff --git a/src/pkg/compress/flate/deflate_test.go b/src/pkg/compress/flate/deflate_test.go index 650a8059a..2ac811c38 100644 --- a/src/pkg/compress/flate/deflate_test.go +++ b/src/pkg/compress/flate/deflate_test.go @@ -57,7 +57,7 @@ var deflateInflateTests = []*deflateInflateTest{ &deflateInflateTest{[]byte{0x11, 0x12}}, &deflateInflateTest{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}}, &deflateInflateTest{[]byte{0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13}}, - &deflateInflateTest{getLargeDataChunk()}, + &deflateInflateTest{largeDataChunk()}, } var reverseBitsTests = []*reverseBitsTest{ @@ -71,23 +71,22 @@ var reverseBitsTests = []*reverseBitsTest{ &reverseBitsTest{29, 5, 23}, } -func getLargeDataChunk() []byte { +func largeDataChunk() []byte { result := make([]byte, 100000) for i := range result { - result[i] = byte(int64(i) * int64(i) & 0xFF) + result[i] = byte(i * i & 0xFF) } return result } func TestDeflate(t *testing.T) { for _, h := range deflateTests { - buffer := bytes.NewBuffer(nil) - w := NewWriter(buffer, h.level) + var buf bytes.Buffer + w := NewWriter(&buf, h.level) w.Write(h.in) w.Close() - if bytes.Compare(buffer.Bytes(), h.out) != 0 { - t.Errorf("buffer is wrong; level = %v, buffer.Bytes() = %v, expected output = %v", - h.level, buffer.Bytes(), h.out) + if !bytes.Equal(buf.Bytes(), h.out) { + t.Errorf("Deflate(%d, %x) = %x, want %x", h.level, h.in, buf.Bytes(), h.out) } } } diff --git a/src/pkg/compress/flate/huffman_bit_writer.go b/src/pkg/compress/flate/huffman_bit_writer.go index abff82dd6..3981df5cb 100644 --- a/src/pkg/compress/flate/huffman_bit_writer.go +++ b/src/pkg/compress/flate/huffman_bit_writer.go @@ -15,9 +15,6 @@ const ( // The largest offset code. offsetCodeCount = 30 - // The largest offset code in the extensions. - extendedOffsetCodeCount = 42 - // The special code used to mark the end of a block. endBlockMarker = 256 @@ -100,11 +97,11 @@ func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter { return &huffmanBitWriter{ w: w, literalFreq: make([]int32, maxLit), - offsetFreq: make([]int32, extendedOffsetCodeCount), - codegen: make([]uint8, maxLit+extendedOffsetCodeCount+1), + offsetFreq: make([]int32, offsetCodeCount), + codegen: make([]uint8, maxLit+offsetCodeCount+1), codegenFreq: make([]int32, codegenCodeCount), literalEncoding: newHuffmanEncoder(maxLit), - offsetEncoding: newHuffmanEncoder(extendedOffsetCodeCount), + offsetEncoding: newHuffmanEncoder(offsetCodeCount), codegenEncoding: newHuffmanEncoder(codegenCodeCount), } } @@ -185,7 +182,7 @@ func (w *huffmanBitWriter) writeBytes(bytes []byte) { _, w.err = w.w.Write(bytes) } -// RFC 1951 3.2.7 specifies a special run-length encoding for specifiying +// RFC 1951 3.2.7 specifies a special run-length encoding for specifying // the literal and offset lengths arrays (which are concatenated into a single // array). This method generates that run-length encoding. // @@ -279,7 +276,7 @@ func (w *huffmanBitWriter) writeCode(code *huffmanEncoder, literal uint32) { // // numLiterals The number of literals specified in codegen // numOffsets The number of offsets specified in codegen -// numCodegens Tne number of codegens used in codegen +// numCodegens The number of codegens used in codegen func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) { if w.err != nil { return @@ -290,13 +287,7 @@ func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, n } w.writeBits(firstBits, 3) w.writeBits(int32(numLiterals-257), 5) - if numOffsets > offsetCodeCount { - // Extended version of decompressor - w.writeBits(int32(offsetCodeCount+((numOffsets-(1+offsetCodeCount))>>3)), 5) - w.writeBits(int32((numOffsets-(1+offsetCodeCount))&0x7), 3) - } else { - w.writeBits(int32(numOffsets-1), 5) - } + w.writeBits(int32(numOffsets-1), 5) w.writeBits(int32(numCodegens-4), 4) for i := 0; i < numCodegens; i++ { @@ -368,24 +359,17 @@ func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) { tokens = tokens[0 : n+1] tokens[n] = endBlockMarker - totalLength := -1 // Subtract 1 for endBlock. for _, t := range tokens { switch t.typ() { case literalType: w.literalFreq[t.literal()]++ - totalLength++ - break case matchType: length := t.length() offset := t.offset() - totalLength += int(length + 3) w.literalFreq[lengthCodesStart+lengthCode(length)]++ w.offsetFreq[offsetCode(offset)]++ - break } } - w.literalEncoding.generate(w.literalFreq, 15) - w.offsetEncoding.generate(w.offsetFreq, 15) // get the number of literals numLiterals := len(w.literalFreq) @@ -394,15 +378,25 @@ func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) { } // get the number of offsets numOffsets := len(w.offsetFreq) - for numOffsets > 1 && w.offsetFreq[numOffsets-1] == 0 { + for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 { numOffsets-- } + if numOffsets == 0 { + // We haven't found a single match. If we want to go with the dynamic encoding, + // we should count at least one offset to be sure that the offset huffman tree could be encoded. + w.offsetFreq[0] = 1 + numOffsets = 1 + } + + w.literalEncoding.generate(w.literalFreq, 15) + w.offsetEncoding.generate(w.offsetFreq, 15) + storedBytes := 0 if input != nil { storedBytes = len(input) } var extraBits int64 - var storedSize int64 + var storedSize int64 = math.MaxInt64 if storedBytes <= maxStoreBlockSize && input != nil { storedSize = int64((storedBytes + 5) * 8) // We only bother calculating the costs of the extra bits required by @@ -417,34 +411,29 @@ func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) { // First four offset codes have extra size = 0. extraBits += int64(w.offsetFreq[offsetCode]) * int64(offsetExtraBits[offsetCode]) } - } else { - storedSize = math.MaxInt32 } - // Figure out which generates smaller code, fixed Huffman, dynamic - // Huffman, or just storing the data. - var fixedSize int64 = math.MaxInt64 - if numOffsets <= offsetCodeCount { - fixedSize = int64(3) + - fixedLiteralEncoding.bitLength(w.literalFreq) + - fixedOffsetEncoding.bitLength(w.offsetFreq) + - extraBits - } + // Figure out smallest code. + // Fixed Huffman baseline. + var size = int64(3) + + fixedLiteralEncoding.bitLength(w.literalFreq) + + fixedOffsetEncoding.bitLength(w.offsetFreq) + + extraBits + var literalEncoding = fixedLiteralEncoding + var offsetEncoding = fixedOffsetEncoding + + // Dynamic Huffman? + var numCodegens int + // Generate codegen and codegenFrequencies, which indicates how to encode // the literalEncoding and the offsetEncoding. w.generateCodegen(numLiterals, numOffsets) w.codegenEncoding.generate(w.codegenFreq, 7) - numCodegens := len(w.codegenFreq) + numCodegens = len(w.codegenFreq) for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { numCodegens-- } - extensionSummand := 0 - if numOffsets > offsetCodeCount { - extensionSummand = 3 - } dynamicHeader := int64(3+5+5+4+(3*numCodegens)) + - // Following line is an extension. - int64(extensionSummand) + w.codegenEncoding.bitLength(w.codegenFreq) + int64(extraBits) + int64(w.codegenFreq[16]*2) + @@ -454,26 +443,25 @@ func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) { w.literalEncoding.bitLength(w.literalFreq) + w.offsetEncoding.bitLength(w.offsetFreq) - if storedSize < fixedSize && storedSize < dynamicSize { + if dynamicSize < size { + size = dynamicSize + literalEncoding = w.literalEncoding + offsetEncoding = w.offsetEncoding + } + + // Stored bytes? + if storedSize < size { w.writeStoredHeader(storedBytes, eof) w.writeBytes(input[0:storedBytes]) return } - var literalEncoding *huffmanEncoder - var offsetEncoding *huffmanEncoder - if fixedSize <= dynamicSize { + // Huffman. + if literalEncoding == fixedLiteralEncoding { w.writeFixedHeader(eof) - literalEncoding = fixedLiteralEncoding - offsetEncoding = fixedOffsetEncoding } else { - // Write the header. w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) - literalEncoding = w.literalEncoding - offsetEncoding = w.offsetEncoding } - - // Write the tokens. for _, t := range tokens { switch t.typ() { case literalType: diff --git a/src/pkg/compress/flate/huffman_code.go b/src/pkg/compress/flate/huffman_code.go index 6be605f0a..7ed603a4f 100644 --- a/src/pkg/compress/flate/huffman_code.go +++ b/src/pkg/compress/flate/huffman_code.go @@ -363,7 +363,12 @@ func (s literalNodeSorter) Less(i, j int) bool { func (s literalNodeSorter) Swap(i, j int) { s.a[i], s.a[j] = s.a[j], s.a[i] } func sortByFreq(a []literalNode) { - s := &literalNodeSorter{a, func(i, j int) bool { return a[i].freq < a[j].freq }} + s := &literalNodeSorter{a, func(i, j int) bool { + if a[i].freq == a[j].freq { + return a[i].literal < a[j].literal + } + return a[i].freq < a[j].freq + }} sort.Sort(s) } diff --git a/src/pkg/compress/flate/inflate.go b/src/pkg/compress/flate/inflate.go index 320b80d06..3845f1204 100644 --- a/src/pkg/compress/flate/inflate.go +++ b/src/pkg/compress/flate/inflate.go @@ -77,8 +77,6 @@ type huffmanDecoder struct { // Initialize Huffman decoding tables from array of code lengths. func (h *huffmanDecoder) init(bits []int) bool { - // TODO(rsc): Return false sometimes. - // Count number of codes of each length, // compute min and max length. var count [maxCodeLen + 1]int @@ -197,9 +195,8 @@ type Reader interface { // Decompress state. type decompressor struct { - // Input/output sources. + // Input source. r Reader - w io.Writer roffset int64 woffset int64 @@ -222,38 +219,79 @@ type decompressor struct { // Temporary buffer (avoids repeated allocation). buf [4]byte + + // Next step in the decompression, + // and decompression state. + step func(*decompressor) + final bool + err os.Error + toRead []byte + hl, hd *huffmanDecoder + copyLen int + copyDist int } -func (f *decompressor) inflate() (err os.Error) { - final := false - for err == nil && !final { - for f.nb < 1+2 { - if err = f.moreBits(); err != nil { - return - } +func (f *decompressor) nextBlock() { + if f.final { + if f.hw != f.hp { + f.flush((*decompressor).nextBlock) + return } - final = f.b&1 == 1 - f.b >>= 1 - typ := f.b & 3 - f.b >>= 2 - f.nb -= 1 + 2 - switch typ { - case 0: - err = f.dataBlock() - case 1: - // compressed, fixed Huffman tables - err = f.decodeBlock(&fixedHuffmanDecoder, nil) - case 2: - // compressed, dynamic Huffman tables - if err = f.readHuffman(); err == nil { - err = f.decodeBlock(&f.h1, &f.h2) - } - default: - // 3 is reserved. - err = CorruptInputError(f.roffset) + f.err = os.EOF + return + } + for f.nb < 1+2 { + if f.err = f.moreBits(); f.err != nil { + return + } + } + f.final = f.b&1 == 1 + f.b >>= 1 + typ := f.b & 3 + f.b >>= 2 + f.nb -= 1 + 2 + switch typ { + case 0: + f.dataBlock() + case 1: + // compressed, fixed Huffman tables + f.hl = &fixedHuffmanDecoder + f.hd = nil + f.huffmanBlock() + case 2: + // compressed, dynamic Huffman tables + if f.err = f.readHuffman(); f.err != nil { + break + } + f.hl = &f.h1 + f.hd = &f.h2 + f.huffmanBlock() + default: + // 3 is reserved. + f.err = CorruptInputError(f.roffset) + } +} + +func (f *decompressor) Read(b []byte) (int, os.Error) { + for { + if len(f.toRead) > 0 { + n := copy(b, f.toRead) + f.toRead = f.toRead[n:] + return n, nil + } + if f.err != nil { + return 0, f.err } + f.step(f) } - return + panic("unreachable") +} + +func (f *decompressor) Close() os.Error { + if f.err == os.EOF { + return nil + } + return f.err } // RFC 1951 section 3.2.7. @@ -358,11 +396,12 @@ func (f *decompressor) readHuffman() os.Error { // hl and hd are the Huffman states for the lit/length values // and the distance values, respectively. If hd == nil, using the // fixed distance encoding associated with fixed Huffman blocks. -func (f *decompressor) decodeBlock(hl, hd *huffmanDecoder) os.Error { +func (f *decompressor) huffmanBlock() { for { - v, err := f.huffSym(hl) + v, err := f.huffSym(f.hl) if err != nil { - return err + f.err = err + return } var n uint // number of bits extra var length int @@ -371,13 +410,15 @@ func (f *decompressor) decodeBlock(hl, hd *huffmanDecoder) os.Error { f.hist[f.hp] = byte(v) f.hp++ if f.hp == len(f.hist) { - if err = f.flush(); err != nil { - return err - } + // After the flush, continue this loop. + f.flush((*decompressor).huffmanBlock) + return } continue case v == 256: - return nil + // Done with huffman block; read next block. + f.step = (*decompressor).nextBlock + return // otherwise, reference to older data case v < 265: length = v - (257 - 3) @@ -404,7 +445,8 @@ func (f *decompressor) decodeBlock(hl, hd *huffmanDecoder) os.Error { if n > 0 { for f.nb < n { if err = f.moreBits(); err != nil { - return err + f.err = err + return } } length += int(f.b & uint32(1<<n-1)) @@ -413,18 +455,20 @@ func (f *decompressor) decodeBlock(hl, hd *huffmanDecoder) os.Error { } var dist int - if hd == nil { + if f.hd == nil { for f.nb < 5 { if err = f.moreBits(); err != nil { - return err + f.err = err + return } } dist = int(reverseByte[(f.b&0x1F)<<3]) f.b >>= 5 f.nb -= 5 } else { - if dist, err = f.huffSym(hd); err != nil { - return err + if dist, err = f.huffSym(f.hd); err != nil { + f.err = err + return } } @@ -432,14 +476,16 @@ func (f *decompressor) decodeBlock(hl, hd *huffmanDecoder) os.Error { case dist < 4: dist++ case dist >= 30: - return CorruptInputError(f.roffset) + f.err = CorruptInputError(f.roffset) + return default: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. extra := (dist & 1) << nb for f.nb < nb { if err = f.moreBits(); err != nil { - return err + f.err = err + return } } extra |= int(f.b & uint32(1<<nb-1)) @@ -450,12 +496,14 @@ func (f *decompressor) decodeBlock(hl, hd *huffmanDecoder) os.Error { // Copy history[-dist:-dist+length] into output. if dist > len(f.hist) { - return InternalError("bad history distance") + f.err = InternalError("bad history distance") + return } // No check on length; encoding can be prescient. if !f.hfull && dist > f.hp { - return CorruptInputError(f.roffset) + f.err = CorruptInputError(f.roffset) + return } p := f.hp - dist @@ -467,9 +515,11 @@ func (f *decompressor) decodeBlock(hl, hd *huffmanDecoder) os.Error { f.hp++ p++ if f.hp == len(f.hist) { - if err = f.flush(); err != nil { - return err - } + // After flush continue copying out of history. + f.copyLen = length - (i + 1) + f.copyDist = dist + f.flush((*decompressor).copyHuff) + return } if p == len(f.hist) { p = 0 @@ -479,8 +529,33 @@ func (f *decompressor) decodeBlock(hl, hd *huffmanDecoder) os.Error { panic("unreached") } +func (f *decompressor) copyHuff() { + length := f.copyLen + dist := f.copyDist + p := f.hp - dist + if p < 0 { + p += len(f.hist) + } + for i := 0; i < length; i++ { + f.hist[f.hp] = f.hist[p] + f.hp++ + p++ + if f.hp == len(f.hist) { + f.copyLen = length - (i + 1) + f.flush((*decompressor).copyHuff) + return + } + if p == len(f.hist) { + p = 0 + } + } + + // Continue processing Huffman block. + f.huffmanBlock() +} + // Copy a single uncompressed data block from input to output. -func (f *decompressor) dataBlock() os.Error { +func (f *decompressor) dataBlock() { // Uncompressed. // Discard current half-byte. f.nb = 0 @@ -490,21 +565,30 @@ func (f *decompressor) dataBlock() os.Error { nr, err := io.ReadFull(f.r, f.buf[0:4]) f.roffset += int64(nr) if err != nil { - return &ReadError{f.roffset, err} + f.err = &ReadError{f.roffset, err} + return } n := int(f.buf[0]) | int(f.buf[1])<<8 nn := int(f.buf[2]) | int(f.buf[3])<<8 if uint16(nn) != uint16(^n) { - return CorruptInputError(f.roffset) + f.err = CorruptInputError(f.roffset) + return } if n == 0 { // 0-length block means sync - return f.flush() + f.flush((*decompressor).nextBlock) + return } - // Read len bytes into history, - // writing as history fills. + f.copyLen = n + f.copyData() +} + +func (f *decompressor) copyData() { + // Read f.dataLen bytes into history, + // pausing for reads as history fills. + n := f.copyLen for n > 0 { m := len(f.hist) - f.hp if m > n { @@ -513,17 +597,18 @@ func (f *decompressor) dataBlock() os.Error { m, err := io.ReadFull(f.r, f.hist[f.hp:f.hp+m]) f.roffset += int64(m) if err != nil { - return &ReadError{f.roffset, err} + f.err = &ReadError{f.roffset, err} + return } n -= m f.hp += m if f.hp == len(f.hist) { - if err = f.flush(); err != nil { - return err - } + f.copyLen = n + f.flush((*decompressor).copyData) + return } } - return nil + f.step = (*decompressor).nextBlock } func (f *decompressor) setDict(dict []byte) { @@ -579,17 +664,8 @@ func (f *decompressor) huffSym(h *huffmanDecoder) (int, os.Error) { } // Flush any buffered output to the underlying writer. -func (f *decompressor) flush() os.Error { - if f.hw == f.hp { - return nil - } - n, err := f.w.Write(f.hist[f.hw:f.hp]) - if n != f.hp-f.hw && err == nil { - err = io.ErrShortWrite - } - if err != nil { - return &WriteError{f.woffset, err} - } +func (f *decompressor) flush(step func(*decompressor)) { + f.toRead = f.hist[f.hw:f.hp] f.woffset += int64(f.hp - f.hw) f.hw = f.hp if f.hp == len(f.hist) { @@ -597,7 +673,7 @@ func (f *decompressor) flush() os.Error { f.hw = 0 f.hfull = true } - return nil + f.step = step } func makeReader(r io.Reader) Reader { @@ -607,30 +683,15 @@ func makeReader(r io.Reader) Reader { return bufio.NewReader(r) } -// decompress reads DEFLATE-compressed data from r and writes -// the uncompressed data to w. -func (f *decompressor) decompress(r io.Reader, w io.Writer) os.Error { - f.r = makeReader(r) - f.w = w - f.woffset = 0 - if err := f.inflate(); err != nil { - return err - } - if err := f.flush(); err != nil { - return err - } - return nil -} - // NewReader returns a new ReadCloser that can be used // to read the uncompressed version of r. It is the caller's // responsibility to call Close on the ReadCloser when // finished reading. func NewReader(r io.Reader) io.ReadCloser { var f decompressor - pr, pw := io.Pipe() - go func() { pw.CloseWithError(f.decompress(r, pw)) }() - return pr + f.r = makeReader(r) + f.step = (*decompressor).nextBlock + return &f } // NewReaderDict is like NewReader but initializes the reader @@ -641,7 +702,7 @@ func NewReader(r io.Reader) io.ReadCloser { func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser { var f decompressor f.setDict(dict) - pr, pw := io.Pipe() - go func() { pw.CloseWithError(f.decompress(r, pw)) }() - return pr + f.r = makeReader(r) + f.step = (*decompressor).nextBlock + return &f } diff --git a/src/pkg/compress/gzip/gzip_test.go b/src/pkg/compress/gzip/gzip_test.go index 23f351405..121e627e6 100644 --- a/src/pkg/compress/gzip/gzip_test.go +++ b/src/pkg/compress/gzip/gzip_test.go @@ -11,7 +11,7 @@ import ( ) // pipe creates two ends of a pipe that gzip and gunzip, and runs dfunc at the -// writer end and ifunc at the reader end. +// writer end and cfunc at the reader end. func pipe(t *testing.T, dfunc func(*Compressor), cfunc func(*Decompressor)) { piper, pipew := io.Pipe() defer piper.Close() diff --git a/src/pkg/compress/lzw/reader.go b/src/pkg/compress/lzw/reader.go index d418bc856..21231c8e5 100644 --- a/src/pkg/compress/lzw/reader.go +++ b/src/pkg/compress/lzw/reader.go @@ -32,13 +32,49 @@ const ( MSB ) +const ( + maxWidth = 12 + decoderInvalidCode = 0xffff + flushBuffer = 1 << maxWidth +) + // decoder is the state from which the readXxx method converts a byte // stream into a code stream. type decoder struct { - r io.ByteReader - bits uint32 - nBits uint - width uint + r io.ByteReader + bits uint32 + nBits uint + width uint + read func(*decoder) (uint16, os.Error) // readLSB or readMSB + litWidth int // width in bits of literal codes + err os.Error + + // The first 1<<litWidth codes are literal codes. + // The next two codes mean clear and EOF. + // Other valid codes are in the range [lo, hi] where lo := clear + 2, + // with the upper bound incrementing on each code seen. + // overflow is the code at which hi overflows the code width. + // last is the most recently seen code, or decoderInvalidCode. + clear, eof, hi, overflow, last uint16 + + // Each code c in [lo, hi] expands to two or more bytes. For c != hi: + // suffix[c] is the last of these bytes. + // prefix[c] is the code for all but the last byte. + // This code can either be a literal code or another code in [lo, c). + // The c == hi case is a special case. + suffix [1 << maxWidth]uint8 + prefix [1 << maxWidth]uint16 + + // output is the temporary output buffer. + // Literal codes are accumulated from the start of the buffer. + // Non-literal codes decode to a sequence of suffixes that are first + // written right-to-left from the end of the buffer before being copied + // to the start of the buffer. + // It is flushed when it contains >= 1<<maxWidth bytes, + // so that there is always room to decode an entire code. + output [2 * 1 << maxWidth]byte + o int // write index into output + toRead []byte // bytes to return from Read } // readLSB returns the next code for "Least Significant Bits first" data. @@ -73,116 +109,113 @@ func (d *decoder) readMSB() (uint16, os.Error) { return code, nil } -// decode decompresses bytes from r and writes them to pw. -// read specifies how to decode bytes into codes. -// litWidth is the width in bits of literal codes. -func decode(r io.Reader, read func(*decoder) (uint16, os.Error), litWidth int, pw *io.PipeWriter) { - br, ok := r.(io.ByteReader) - if !ok { - br = bufio.NewReader(r) +func (d *decoder) Read(b []byte) (int, os.Error) { + for { + if len(d.toRead) > 0 { + n := copy(b, d.toRead) + d.toRead = d.toRead[n:] + return n, nil + } + if d.err != nil { + return 0, d.err + } + d.decode() } - pw.CloseWithError(decode1(pw, br, read, uint(litWidth))) + panic("unreachable") } -func decode1(pw *io.PipeWriter, r io.ByteReader, read func(*decoder) (uint16, os.Error), litWidth uint) os.Error { - const ( - maxWidth = 12 - invalidCode = 0xffff - ) - d := decoder{r, 0, 0, 1 + litWidth} - w := bufio.NewWriter(pw) - // The first 1<<litWidth codes are literal codes. - // The next two codes mean clear and EOF. - // Other valid codes are in the range [lo, hi] where lo := clear + 2, - // with the upper bound incrementing on each code seen. - clear := uint16(1) << litWidth - eof, hi := clear+1, clear+1 - // overflow is the code at which hi overflows the code width. - overflow := uint16(1) << d.width - var ( - // Each code c in [lo, hi] expands to two or more bytes. For c != hi: - // suffix[c] is the last of these bytes. - // prefix[c] is the code for all but the last byte. - // This code can either be a literal code or another code in [lo, c). - // The c == hi case is a special case. - suffix [1 << maxWidth]uint8 - prefix [1 << maxWidth]uint16 - // buf is a scratch buffer for reconstituting the bytes that a code expands to. - // Code suffixes are written right-to-left from the end of the buffer. - buf [1 << maxWidth]byte - ) - +// decode decompresses bytes from r and leaves them in d.toRead. +// read specifies how to decode bytes into codes. +// litWidth is the width in bits of literal codes. +func (d *decoder) decode() { // Loop over the code stream, converting codes into decompressed bytes. - last := uint16(invalidCode) for { - code, err := read(&d) + code, err := d.read(d) if err != nil { if err == os.EOF { err = io.ErrUnexpectedEOF } - return err + d.err = err + return } switch { - case code < clear: + case code < d.clear: // We have a literal code. - if err := w.WriteByte(uint8(code)); err != nil { - return err - } - if last != invalidCode { + d.output[d.o] = uint8(code) + d.o++ + if d.last != decoderInvalidCode { // Save what the hi code expands to. - suffix[hi] = uint8(code) - prefix[hi] = last + d.suffix[d.hi] = uint8(code) + d.prefix[d.hi] = d.last } - case code == clear: - d.width = 1 + litWidth - hi = eof - overflow = 1 << d.width - last = invalidCode + case code == d.clear: + d.width = 1 + uint(d.litWidth) + d.hi = d.eof + d.overflow = 1 << d.width + d.last = decoderInvalidCode continue - case code == eof: - return w.Flush() - case code <= hi: - c, i := code, len(buf)-1 - if code == hi { + case code == d.eof: + d.flush() + d.err = os.EOF + return + case code <= d.hi: + c, i := code, len(d.output)-1 + if code == d.hi { // code == hi is a special case which expands to the last expansion // followed by the head of the last expansion. To find the head, we walk // the prefix chain until we find a literal code. - c = last - for c >= clear { - c = prefix[c] + c = d.last + for c >= d.clear { + c = d.prefix[c] } - buf[i] = uint8(c) + d.output[i] = uint8(c) i-- - c = last + c = d.last } - // Copy the suffix chain into buf and then write that to w. - for c >= clear { - buf[i] = suffix[c] + // Copy the suffix chain into output and then write that to w. + for c >= d.clear { + d.output[i] = d.suffix[c] i-- - c = prefix[c] + c = d.prefix[c] } - buf[i] = uint8(c) - if _, err := w.Write(buf[i:]); err != nil { - return err + d.output[i] = uint8(c) + d.o += copy(d.output[d.o:], d.output[i:]) + if d.last != decoderInvalidCode { + // Save what the hi code expands to. + d.suffix[d.hi] = uint8(c) + d.prefix[d.hi] = d.last } - // Save what the hi code expands to. - suffix[hi] = uint8(c) - prefix[hi] = last default: - return os.NewError("lzw: invalid code") + d.err = os.NewError("lzw: invalid code") + return } - last, hi = code, hi+1 - if hi == overflow { + d.last, d.hi = code, d.hi+1 + if d.hi >= d.overflow { if d.width == maxWidth { - return os.NewError("lzw: missing clear code") + d.last = decoderInvalidCode + } else { + d.width++ + d.overflow <<= 1 } - d.width++ - overflow <<= 1 + } + if d.o >= flushBuffer { + d.flush() + return } } panic("unreachable") } +func (d *decoder) flush() { + d.toRead = d.output[:d.o] + d.o = 0 +} + +func (d *decoder) Close() os.Error { + d.err = os.EINVAL // in case any Reads come along + return nil +} + // NewReader creates a new io.ReadCloser that satisfies reads by decompressing // the data read from r. // It is the caller's responsibility to call Close on the ReadCloser when @@ -190,21 +223,31 @@ func decode1(pw *io.PipeWriter, r io.ByteReader, read func(*decoder) (uint16, os // The number of bits to use for literal codes, litWidth, must be in the // range [2,8] and is typically 8. func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser { - pr, pw := io.Pipe() - var read func(*decoder) (uint16, os.Error) + d := new(decoder) switch order { case LSB: - read = (*decoder).readLSB + d.read = (*decoder).readLSB case MSB: - read = (*decoder).readMSB + d.read = (*decoder).readMSB default: - pw.CloseWithError(os.NewError("lzw: unknown order")) - return pr + d.err = os.NewError("lzw: unknown order") + return d } if litWidth < 2 || 8 < litWidth { - pw.CloseWithError(fmt.Errorf("lzw: litWidth %d out of range", litWidth)) - return pr + d.err = fmt.Errorf("lzw: litWidth %d out of range", litWidth) + return d } - go decode(r, read, litWidth, pw) - return pr + if br, ok := r.(io.ByteReader); ok { + d.r = br + } else { + d.r = bufio.NewReader(r) + } + d.litWidth = litWidth + d.width = 1 + uint(litWidth) + d.clear = uint16(1) << uint(litWidth) + d.eof, d.hi = d.clear+1, d.clear+1 + d.overflow = uint16(1) << d.width + d.last = decoderInvalidCode + + return d } diff --git a/src/pkg/compress/lzw/writer_test.go b/src/pkg/compress/lzw/writer_test.go index 82464ecd1..4c5e522f9 100644 --- a/src/pkg/compress/lzw/writer_test.go +++ b/src/pkg/compress/lzw/writer_test.go @@ -77,13 +77,13 @@ func testFile(t *testing.T, fn string, order Order, litWidth int) { t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err1) return } - if len(b0) != len(b1) { - t.Errorf("%s (order=%d litWidth=%d): length mismatch %d versus %d", fn, order, litWidth, len(b0), len(b1)) + if len(b1) != len(b0) { + t.Errorf("%s (order=%d litWidth=%d): length mismatch %d != %d", fn, order, litWidth, len(b1), len(b0)) return } for i := 0; i < len(b0); i++ { - if b0[i] != b1[i] { - t.Errorf("%s (order=%d litWidth=%d): mismatch at %d, 0x%02x versus 0x%02x\n", fn, order, litWidth, i, b0[i], b1[i]) + if b1[i] != b0[i] { + t.Errorf("%s (order=%d litWidth=%d): mismatch at %d, 0x%02x != 0x%02x\n", fn, order, litWidth, i, b1[i], b0[i]) return } } diff --git a/src/pkg/compress/zlib/writer.go b/src/pkg/compress/zlib/writer.go index f1f9b2853..8f86e9c4c 100644 --- a/src/pkg/compress/zlib/writer.go +++ b/src/pkg/compress/zlib/writer.go @@ -89,7 +89,7 @@ func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, os.Error) { } } z.w = w - z.compressor = flate.NewWriter(w, level) + z.compressor = flate.NewWriterDict(w, level, dict) z.digest = adler32.New() return z, nil } diff --git a/src/pkg/compress/zlib/writer_test.go b/src/pkg/compress/zlib/writer_test.go index f94f28470..32f05ab68 100644 --- a/src/pkg/compress/zlib/writer_test.go +++ b/src/pkg/compress/zlib/writer_test.go @@ -5,6 +5,8 @@ package zlib import ( + "bytes" + "fmt" "io" "io/ioutil" "os" @@ -16,15 +18,13 @@ var filenames = []string{ "../testdata/pi.txt", } +var data = []string{ + "test a reasonable sized string that can be compressed", +} + // Tests that compressing and then decompressing the given file at the given compression level and dictionary // yields equivalent bytes to the original file. func testFileLevelDict(t *testing.T, fn string, level int, d string) { - // Read dictionary, if given. - var dict []byte - if d != "" { - dict = []byte(d) - } - // Read the file, as golden output. golden, err := os.Open(fn) if err != nil { @@ -32,17 +32,25 @@ func testFileLevelDict(t *testing.T, fn string, level int, d string) { return } defer golden.Close() - - // Read the file again, and push it through a pipe that compresses at the write end, and decompresses at the read end. - raw, err := os.Open(fn) - if err != nil { - t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + b0, err0 := ioutil.ReadAll(golden) + if err0 != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err0) return } + testLevelDict(t, fn, b0, level, d) +} + +func testLevelDict(t *testing.T, fn string, b0 []byte, level int, d string) { + // Make dictionary, if given. + var dict []byte + if d != "" { + dict = []byte(d) + } + + // Push data through a pipe that compresses at the write end, and decompresses at the read end. piper, pipew := io.Pipe() defer piper.Close() go func() { - defer raw.Close() defer pipew.Close() zlibw, err := NewWriterDict(pipew, level, dict) if err != nil { @@ -50,25 +58,14 @@ func testFileLevelDict(t *testing.T, fn string, level int, d string) { return } defer zlibw.Close() - var b [1024]byte - for { - n, err0 := raw.Read(b[0:]) - if err0 != nil && err0 != os.EOF { - t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err0) - return - } - _, err1 := zlibw.Write(b[0:n]) - if err1 == os.EPIPE { - // Fail, but do not report the error, as some other (presumably reportable) error broke the pipe. - return - } - if err1 != nil { - t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err1) - return - } - if err0 == os.EOF { - break - } + _, err = zlibw.Write(b0) + if err == os.EPIPE { + // Fail, but do not report the error, as some other (presumably reported) error broke the pipe. + return + } + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + return } }() zlibr, err := NewReaderDict(piper, dict) @@ -78,13 +75,8 @@ func testFileLevelDict(t *testing.T, fn string, level int, d string) { } defer zlibr.Close() - // Compare the two. - b0, err0 := ioutil.ReadAll(golden) + // Compare the decompressed data. b1, err1 := ioutil.ReadAll(zlibr) - if err0 != nil { - t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err0) - return - } if err1 != nil { t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err1) return @@ -102,6 +94,18 @@ func testFileLevelDict(t *testing.T, fn string, level int, d string) { } func TestWriter(t *testing.T) { + for i, s := range data { + b := []byte(s) + tag := fmt.Sprintf("#%d", i) + testLevelDict(t, tag, b, DefaultCompression, "") + testLevelDict(t, tag, b, NoCompression, "") + for level := BestSpeed; level <= BestCompression; level++ { + testLevelDict(t, tag, b, level, "") + } + } +} + +func TestWriterBig(t *testing.T) { for _, fn := range filenames { testFileLevelDict(t, fn, DefaultCompression, "") testFileLevelDict(t, fn, NoCompression, "") @@ -121,3 +125,20 @@ func TestWriterDict(t *testing.T) { } } } + +func TestWriterDictIsUsed(t *testing.T) { + var input = []byte("Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.") + buf := bytes.NewBuffer(nil) + compressor, err := NewWriterDict(buf, BestCompression, input) + if err != nil { + t.Errorf("error in NewWriterDict: %s", err) + return + } + compressor.Write(input) + compressor.Close() + const expectedMaxSize = 25 + output := buf.Bytes() + if len(output) > expectedMaxSize { + t.Errorf("result too large (got %d, want <= %d bytes). Is the dictionary being used?", len(output), expectedMaxSize) + } +} diff --git a/src/pkg/container/heap/heap.go b/src/pkg/container/heap/heap.go index f2b8a750a..5b68827df 100644 --- a/src/pkg/container/heap/heap.go +++ b/src/pkg/container/heap/heap.go @@ -22,7 +22,7 @@ type Interface interface { } -// A heaper must be initialized before any of the heap operations +// A heap must be initialized before any of the heap operations // can be used. Init is idempotent with respect to the heap invariants // and may be called whenever the heap invariants may have been invalidated. // Its complexity is O(n) where n = h.Len(). diff --git a/src/pkg/crypto/cipher/ocfb.go b/src/pkg/crypto/cipher/ocfb.go index b2d877591..031e74a9d 100644 --- a/src/pkg/crypto/cipher/ocfb.go +++ b/src/pkg/crypto/cipher/ocfb.go @@ -80,9 +80,10 @@ type ocfbDecrypter struct { // NewOCFBDecrypter returns a Stream which decrypts data with OpenPGP's cipher // feedback mode using the given Block. Prefix must be the first blockSize + 2 // bytes of the ciphertext, where blockSize is the Block's block size. If an -// incorrect key is detected then nil is returned. Resync determines if the -// "resynchronization step" from RFC 4880, 13.9 step 7 is performed. Different -// parts of OpenPGP vary on this point. +// incorrect key is detected then nil is returned. On successful exit, +// blockSize+2 bytes of decrypted data are written into prefix. Resync +// determines if the "resynchronization step" from RFC 4880, 13.9 step 7 is +// performed. Different parts of OpenPGP vary on this point. func NewOCFBDecrypter(block Block, prefix []byte, resync OCFBResyncOption) Stream { blockSize := block.BlockSize() if len(prefix) != blockSize+2 { @@ -118,6 +119,7 @@ func NewOCFBDecrypter(block Block, prefix []byte, resync OCFBResyncOption) Strea x.fre[1] = prefix[blockSize+1] x.outUsed = 2 } + copy(prefix, prefixCopy) return x } diff --git a/src/pkg/crypto/elliptic/elliptic.go b/src/pkg/crypto/elliptic/elliptic.go index 335c9645d..41835f1a9 100644 --- a/src/pkg/crypto/elliptic/elliptic.go +++ b/src/pkg/crypto/elliptic/elliptic.go @@ -284,7 +284,7 @@ func (curve *Curve) Marshal(x, y *big.Int) []byte { return ret } -// Unmarshal converts a point, serialised by Marshal, into an x, y pair. On +// Unmarshal converts a point, serialized by Marshal, into an x, y pair. On // error, x = nil. func (curve *Curve) Unmarshal(data []byte) (x, y *big.Int) { byteLen := (curve.BitSize + 7) >> 3 diff --git a/src/pkg/crypto/elliptic/elliptic_test.go b/src/pkg/crypto/elliptic/elliptic_test.go index 02083a986..b7e7f035f 100644 --- a/src/pkg/crypto/elliptic/elliptic_test.go +++ b/src/pkg/crypto/elliptic/elliptic_test.go @@ -321,8 +321,8 @@ func TestMarshal(t *testing.T) { t.Error(err) return } - serialised := p224.Marshal(x, y) - xx, yy := p224.Unmarshal(serialised) + serialized := p224.Marshal(x, y) + xx, yy := p224.Unmarshal(serialized) if xx == nil { t.Error("failed to unmarshal") return diff --git a/src/pkg/crypto/hmac/hmac_test.go b/src/pkg/crypto/hmac/hmac_test.go index 40adbad04..bcae63b8a 100644 --- a/src/pkg/crypto/hmac/hmac_test.go +++ b/src/pkg/crypto/hmac/hmac_test.go @@ -190,7 +190,7 @@ func TestHMAC(t *testing.T) { continue } - // Repetive Sum() calls should return the same value + // Repetitive Sum() calls should return the same value for k := 0; k < 2; k++ { sum := fmt.Sprintf("%x", h.Sum()) if sum != tt.out { diff --git a/src/pkg/crypto/ocsp/ocsp.go b/src/pkg/crypto/ocsp/ocsp.go index acd75b8b0..57dbe7d2d 100644 --- a/src/pkg/crypto/ocsp/ocsp.go +++ b/src/pkg/crypto/ocsp/ocsp.go @@ -13,6 +13,7 @@ import ( "crypto/rsa" _ "crypto/sha1" "crypto/x509" + "crypto/x509/pkix" "os" "time" ) @@ -32,21 +33,9 @@ const ( ocspUnauthorized = 5 ) -type rdnSequence []relativeDistinguishedNameSET - -type relativeDistinguishedNameSET []attributeTypeAndValue - -type attributeTypeAndValue struct { - Type asn1.ObjectIdentifier - Value interface{} -} - -type algorithmIdentifier struct { - Algorithm asn1.ObjectIdentifier -} type certID struct { - HashAlgorithm algorithmIdentifier + HashAlgorithm pkix.AlgorithmIdentifier NameHash []byte IssuerKeyHash []byte SerialNumber asn1.RawValue @@ -64,16 +53,16 @@ type responseBytes struct { type basicResponse struct { TBSResponseData responseData - SignatureAlgorithm algorithmIdentifier + SignatureAlgorithm pkix.AlgorithmIdentifier Signature asn1.BitString Certificates []asn1.RawValue "explicit,tag:0,optional" } type responseData struct { Raw asn1.RawContent - Version int "optional,default:1,explicit,tag:0" - RequestorName rdnSequence "optional,explicit,tag:1" - KeyHash []byte "optional,explicit,tag:2" + Version int "optional,default:1,explicit,tag:0" + RequestorName pkix.RDNSequence "optional,explicit,tag:1" + KeyHash []byte "optional,explicit,tag:2" ProducedAt *time.Time Responses []singleResponse } diff --git a/src/pkg/crypto/openpgp/armor/armor.go b/src/pkg/crypto/openpgp/armor/armor.go index 8da612c50..9c4180d6d 100644 --- a/src/pkg/crypto/openpgp/armor/armor.go +++ b/src/pkg/crypto/openpgp/armor/armor.go @@ -153,7 +153,7 @@ func (r *openpgpReader) Read(p []byte) (n int, err os.Error) { // Decode reads a PGP armored block from the given Reader. It will ignore // leading garbage. If it doesn't find a block, it will return nil, os.EOF. The -// given Reader is not usable after calling this function: an arbitary amount +// given Reader is not usable after calling this function: an arbitrary amount // of data may have been read past the end of the block. func Decode(in io.Reader) (p *Block, err os.Error) { r, _ := bufio.NewReaderSize(in, 100) diff --git a/src/pkg/crypto/openpgp/keys.go b/src/pkg/crypto/openpgp/keys.go index 6c03f8828..2acb7e612 100644 --- a/src/pkg/crypto/openpgp/keys.go +++ b/src/pkg/crypto/openpgp/keys.go @@ -5,9 +5,11 @@ package openpgp import ( + "crypto" "crypto/openpgp/armor" "crypto/openpgp/error" "crypto/openpgp/packet" + "crypto/rsa" "io" "os" ) @@ -297,3 +299,104 @@ func addSubkey(e *Entity, packets *packet.Reader, pub *packet.PublicKey, priv *p e.Subkeys = append(e.Subkeys, subKey) return nil } + +const defaultRSAKeyBits = 2048 + +// NewEntity returns an Entity that contains a fresh RSA/RSA keypair with a +// single identity composed of the given full name, comment and email, any of +// which may be empty but must not contain any of "()<>\x00". +func NewEntity(rand io.Reader, currentTimeSecs int64, name, comment, email string) (*Entity, os.Error) { + uid := packet.NewUserId(name, comment, email) + if uid == nil { + return nil, error.InvalidArgumentError("user id field contained invalid characters") + } + signingPriv, err := rsa.GenerateKey(rand, defaultRSAKeyBits) + if err != nil { + return nil, err + } + encryptingPriv, err := rsa.GenerateKey(rand, defaultRSAKeyBits) + if err != nil { + return nil, err + } + + t := uint32(currentTimeSecs) + + e := &Entity{ + PrimaryKey: packet.NewRSAPublicKey(t, &signingPriv.PublicKey, false /* not a subkey */ ), + PrivateKey: packet.NewRSAPrivateKey(t, signingPriv, false /* not a subkey */ ), + Identities: make(map[string]*Identity), + } + isPrimaryId := true + e.Identities[uid.Id] = &Identity{ + Name: uid.Name, + UserId: uid, + SelfSignature: &packet.Signature{ + CreationTime: t, + SigType: packet.SigTypePositiveCert, + PubKeyAlgo: packet.PubKeyAlgoRSA, + Hash: crypto.SHA256, + IsPrimaryId: &isPrimaryId, + FlagsValid: true, + FlagSign: true, + FlagCertify: true, + IssuerKeyId: &e.PrimaryKey.KeyId, + }, + } + + e.Subkeys = make([]Subkey, 1) + e.Subkeys[0] = Subkey{ + PublicKey: packet.NewRSAPublicKey(t, &encryptingPriv.PublicKey, true /* is a subkey */ ), + PrivateKey: packet.NewRSAPrivateKey(t, encryptingPriv, true /* is a subkey */ ), + Sig: &packet.Signature{ + CreationTime: t, + SigType: packet.SigTypeSubkeyBinding, + PubKeyAlgo: packet.PubKeyAlgoRSA, + Hash: crypto.SHA256, + FlagsValid: true, + FlagEncryptStorage: true, + FlagEncryptCommunications: true, + IssuerKeyId: &e.PrimaryKey.KeyId, + }, + } + + return e, nil +} + +// SerializePrivate serializes an Entity, including private key material, to +// the given Writer. For now, it must only be used on an Entity returned from +// NewEntity. +func (e *Entity) SerializePrivate(w io.Writer) (err os.Error) { + err = e.PrivateKey.Serialize(w) + if err != nil { + return + } + for _, ident := range e.Identities { + err = ident.UserId.Serialize(w) + if err != nil { + return + } + err = ident.SelfSignature.SignUserId(ident.UserId.Id, e.PrimaryKey, e.PrivateKey) + if err != nil { + return + } + err = ident.SelfSignature.Serialize(w) + if err != nil { + return + } + } + for _, subkey := range e.Subkeys { + err = subkey.PrivateKey.Serialize(w) + if err != nil { + return + } + err = subkey.Sig.SignKey(subkey.PublicKey, e.PrivateKey) + if err != nil { + return + } + err = subkey.Sig.Serialize(w) + if err != nil { + return + } + } + return nil +} diff --git a/src/pkg/crypto/openpgp/packet/literal.go b/src/pkg/crypto/openpgp/packet/literal.go index 04f50e53e..9411572d7 100644 --- a/src/pkg/crypto/openpgp/packet/literal.go +++ b/src/pkg/crypto/openpgp/packet/literal.go @@ -51,3 +51,40 @@ func (l *LiteralData) parse(r io.Reader) (err os.Error) { l.Body = r return } + +// SerializeLiteral serializes a literal data packet to w and returns a +// WriteCloser to which the data itself can be written and which MUST be closed +// on completion. The fileName is truncated to 255 bytes. +func SerializeLiteral(w io.WriteCloser, isBinary bool, fileName string, time uint32) (plaintext io.WriteCloser, err os.Error) { + var buf [4]byte + buf[0] = 't' + if isBinary { + buf[0] = 'b' + } + if len(fileName) > 255 { + fileName = fileName[:255] + } + buf[1] = byte(len(fileName)) + + inner, err := serializeStreamHeader(w, packetTypeLiteralData) + if err != nil { + return + } + + _, err = inner.Write(buf[:2]) + if err != nil { + return + } + _, err = inner.Write([]byte(fileName)) + if err != nil { + return + } + binary.BigEndian.PutUint32(buf[:], time) + _, err = inner.Write(buf[:]) + if err != nil { + return + } + + plaintext = inner + return +} diff --git a/src/pkg/crypto/openpgp/packet/packet.go b/src/pkg/crypto/openpgp/packet/packet.go index c0ec44dd8..640a5b76f 100644 --- a/src/pkg/crypto/openpgp/packet/packet.go +++ b/src/pkg/crypto/openpgp/packet/packet.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package packet implements parsing and serialisation of OpenPGP packets, as +// Package packet implements parsing and serialization of OpenPGP packets, as // specified in RFC 4880. package packet @@ -92,6 +92,46 @@ func (r *partialLengthReader) Read(p []byte) (n int, err os.Error) { return } +// partialLengthWriter writes a stream of data using OpenPGP partial lengths. +// See RFC 4880, section 4.2.2.4. +type partialLengthWriter struct { + w io.WriteCloser + lengthByte [1]byte +} + +func (w *partialLengthWriter) Write(p []byte) (n int, err os.Error) { + for len(p) > 0 { + for power := uint(14); power < 32; power-- { + l := 1 << power + if len(p) >= l { + w.lengthByte[0] = 224 + uint8(power) + _, err = w.w.Write(w.lengthByte[:]) + if err != nil { + return + } + var m int + m, err = w.w.Write(p[:l]) + n += m + if err != nil { + return + } + p = p[l:] + break + } + } + } + return +} + +func (w *partialLengthWriter) Close() os.Error { + w.lengthByte[0] = 0 + _, err := w.w.Write(w.lengthByte[:]) + if err != nil { + return err + } + return w.w.Close() +} + // A spanReader is an io.LimitReader, but it returns ErrUnexpectedEOF if the // underlying Reader returns EOF before the limit has been reached. type spanReader struct { @@ -195,6 +235,20 @@ func serializeHeader(w io.Writer, ptype packetType, length int) (err os.Error) { return } +// serializeStreamHeader writes an OpenPGP packet header to w where the +// length of the packet is unknown. It returns a io.WriteCloser which can be +// used to write the contents of the packet. See RFC 4880, section 4.2. +func serializeStreamHeader(w io.WriteCloser, ptype packetType) (out io.WriteCloser, err os.Error) { + var buf [1]byte + buf[0] = 0x80 | 0x40 | byte(ptype) + _, err = w.Write(buf[:]) + if err != nil { + return + } + out = &partialLengthWriter{w: w} + return +} + // Packet represents an OpenPGP packet. Users are expected to try casting // instances of this interface to specific packet types. type Packet interface { @@ -301,12 +355,12 @@ type SignatureType uint8 const ( SigTypeBinary SignatureType = 0 - SigTypeText = 1 - SigTypeGenericCert = 0x10 - SigTypePersonaCert = 0x11 - SigTypeCasualCert = 0x12 - SigTypePositiveCert = 0x13 - SigTypeSubkeyBinding = 0x18 + SigTypeText = 1 + SigTypeGenericCert = 0x10 + SigTypePersonaCert = 0x11 + SigTypeCasualCert = 0x12 + SigTypePositiveCert = 0x13 + SigTypeSubkeyBinding = 0x18 ) // PublicKeyAlgorithm represents the different public key system specified for @@ -327,10 +381,10 @@ const ( type CipherFunction uint8 const ( - CipherCAST5 = 3 - CipherAES128 = 7 - CipherAES192 = 8 - CipherAES256 = 9 + CipherCAST5 CipherFunction = 3 + CipherAES128 CipherFunction = 7 + CipherAES192 CipherFunction = 8 + CipherAES256 CipherFunction = 9 ) // keySize returns the key size, in bytes, of cipher. @@ -386,6 +440,14 @@ func readMPI(r io.Reader) (mpi []byte, bitLength uint16, err os.Error) { return } +// mpiLength returns the length of the given *big.Int when serialized as an +// MPI. +func mpiLength(n *big.Int) (mpiLengthInBytes int) { + mpiLengthInBytes = 2 /* MPI length */ + mpiLengthInBytes += (n.BitLen() + 7) / 8 + return +} + // writeMPI serializes a big integer to w. func writeMPI(w io.Writer, bitLength uint16, mpiBytes []byte) (err os.Error) { _, err = w.Write([]byte{byte(bitLength >> 8), byte(bitLength)}) diff --git a/src/pkg/crypto/openpgp/packet/packet_test.go b/src/pkg/crypto/openpgp/packet/packet_test.go index 1a4692cd4..23d9978ae 100644 --- a/src/pkg/crypto/openpgp/packet/packet_test.go +++ b/src/pkg/crypto/openpgp/packet/packet_test.go @@ -210,3 +210,47 @@ func TestSerializeHeader(t *testing.T) { } } } + +func TestPartialLengths(t *testing.T) { + buf := bytes.NewBuffer(nil) + w := new(partialLengthWriter) + w.w = noOpCloser{buf} + + const maxChunkSize = 64 + + var b [maxChunkSize]byte + var n uint8 + for l := 1; l <= maxChunkSize; l++ { + for i := 0; i < l; i++ { + b[i] = n + n++ + } + m, err := w.Write(b[:l]) + if m != l { + t.Errorf("short write got: %d want: %d", m, l) + } + if err != nil { + t.Errorf("error from write: %s", err) + } + } + w.Close() + + want := (maxChunkSize * (maxChunkSize + 1)) / 2 + copyBuf := bytes.NewBuffer(nil) + r := &partialLengthReader{buf, 0, true} + m, err := io.Copy(copyBuf, r) + if m != int64(want) { + t.Errorf("short copy got: %d want: %d", m, want) + } + if err != nil { + t.Errorf("error from copy: %s", err) + } + + copyBytes := copyBuf.Bytes() + for i := 0; i < want; i++ { + if copyBytes[i] != uint8(i) { + t.Errorf("bad pattern in copy at %d", i) + break + } + } +} diff --git a/src/pkg/crypto/openpgp/packet/private_key.go b/src/pkg/crypto/openpgp/packet/private_key.go index fde2a9933..92e7ee422 100644 --- a/src/pkg/crypto/openpgp/packet/private_key.go +++ b/src/pkg/crypto/openpgp/packet/private_key.go @@ -32,6 +32,13 @@ type PrivateKey struct { iv []byte } +func NewRSAPrivateKey(currentTimeSecs uint32, priv *rsa.PrivateKey, isSubkey bool) *PrivateKey { + pk := new(PrivateKey) + pk.PublicKey = *NewRSAPublicKey(currentTimeSecs, &priv.PublicKey, isSubkey) + pk.PrivateKey = priv + return pk +} + func (pk *PrivateKey) parse(r io.Reader) (err os.Error) { err = (&pk.PublicKey).parse(r) if err != nil { @@ -91,6 +98,83 @@ func (pk *PrivateKey) parse(r io.Reader) (err os.Error) { return } +func mod64kHash(d []byte) uint16 { + h := uint16(0) + for i := 0; i < len(d); i += 2 { + v := uint16(d[i]) << 8 + if i+1 < len(d) { + v += uint16(d[i+1]) + } + h += v + } + return h +} + +func (pk *PrivateKey) Serialize(w io.Writer) (err os.Error) { + // TODO(agl): support encrypted private keys + buf := bytes.NewBuffer(nil) + err = pk.PublicKey.serializeWithoutHeaders(buf) + if err != nil { + return + } + buf.WriteByte(0 /* no encryption */ ) + + privateKeyBuf := bytes.NewBuffer(nil) + + switch priv := pk.PrivateKey.(type) { + case *rsa.PrivateKey: + err = serializeRSAPrivateKey(privateKeyBuf, priv) + default: + err = error.InvalidArgumentError("non-RSA private key") + } + if err != nil { + return + } + + ptype := packetTypePrivateKey + contents := buf.Bytes() + privateKeyBytes := privateKeyBuf.Bytes() + if pk.IsSubkey { + ptype = packetTypePrivateSubkey + } + err = serializeHeader(w, ptype, len(contents)+len(privateKeyBytes)+2) + if err != nil { + return + } + _, err = w.Write(contents) + if err != nil { + return + } + _, err = w.Write(privateKeyBytes) + if err != nil { + return + } + + checksum := mod64kHash(privateKeyBytes) + var checksumBytes [2]byte + checksumBytes[0] = byte(checksum >> 8) + checksumBytes[1] = byte(checksum) + _, err = w.Write(checksumBytes[:]) + + return +} + +func serializeRSAPrivateKey(w io.Writer, priv *rsa.PrivateKey) os.Error { + err := writeBig(w, priv.D) + if err != nil { + return err + } + err = writeBig(w, priv.Primes[1]) + if err != nil { + return err + } + err = writeBig(w, priv.Primes[0]) + if err != nil { + return err + } + return writeBig(w, priv.Precomputed.Qinv) +} + // Decrypt decrypts an encrypted private key using a passphrase. func (pk *PrivateKey) Decrypt(passphrase []byte) os.Error { if !pk.Encrypted { diff --git a/src/pkg/crypto/openpgp/packet/public_key.go b/src/pkg/crypto/openpgp/packet/public_key.go index cd4a9aebb..46d365b2a 100644 --- a/src/pkg/crypto/openpgp/packet/public_key.go +++ b/src/pkg/crypto/openpgp/packet/public_key.go @@ -30,6 +30,28 @@ type PublicKey struct { n, e, p, q, g, y parsedMPI } +func fromBig(n *big.Int) parsedMPI { + return parsedMPI{ + bytes: n.Bytes(), + bitLength: uint16(n.BitLen()), + } +} + +// NewRSAPublicKey returns a PublicKey that wraps the given rsa.PublicKey. +func NewRSAPublicKey(creationTimeSecs uint32, pub *rsa.PublicKey, isSubkey bool) *PublicKey { + pk := &PublicKey{ + CreationTime: creationTimeSecs, + PubKeyAlgo: PubKeyAlgoRSA, + PublicKey: pub, + IsSubkey: isSubkey, + n: fromBig(pub.N), + e: fromBig(big.NewInt(int64(pub.E))), + } + + pk.setFingerPrintAndKeyId() + return pk +} + func (pk *PublicKey) parse(r io.Reader) (err os.Error) { // RFC 4880, section 5.5.2 var buf [6]byte @@ -54,14 +76,17 @@ func (pk *PublicKey) parse(r io.Reader) (err os.Error) { return } + pk.setFingerPrintAndKeyId() + return +} + +func (pk *PublicKey) setFingerPrintAndKeyId() { // RFC 4880, section 12.2 fingerPrint := sha1.New() pk.SerializeSignaturePrefix(fingerPrint) - pk.Serialize(fingerPrint) + pk.serializeWithoutHeaders(fingerPrint) copy(pk.Fingerprint[:], fingerPrint.Sum()) pk.KeyId = binary.BigEndian.Uint64(pk.Fingerprint[12:20]) - - return } // parseRSA parses RSA public key material from the given Reader. See RFC 4880, @@ -143,9 +168,30 @@ func (pk *PublicKey) SerializeSignaturePrefix(h hash.Hash) { return } -// Serialize marshals the PublicKey to w in the form of an OpenPGP public key -// packet, not including the packet header. func (pk *PublicKey) Serialize(w io.Writer) (err os.Error) { + length := 6 // 6 byte header + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + length += 2 + len(pk.n.bytes) + length += 2 + len(pk.e.bytes) + case PubKeyAlgoDSA: + length += 2 + len(pk.p.bytes) + length += 2 + len(pk.q.bytes) + length += 2 + len(pk.g.bytes) + length += 2 + len(pk.y.bytes) + } + + err = serializeHeader(w, packetTypePublicKey, length) + if err != nil { + return + } + return pk.serializeWithoutHeaders(w) +} + +// serializeWithoutHeaders marshals the PublicKey to w in the form of an +// OpenPGP public key packet, not including the packet header. +func (pk *PublicKey) serializeWithoutHeaders(w io.Writer) (err os.Error) { var buf [6]byte buf[0] = 4 buf[1] = byte(pk.CreationTime >> 24) @@ -211,34 +257,43 @@ func (pk *PublicKey) VerifySignature(signed hash.Hash, sig *Signature) (err os.E panic("unreachable") } -// VerifyKeySignature returns nil iff sig is a valid signature, make by this -// public key, of the public key in signed. -func (pk *PublicKey) VerifyKeySignature(signed *PublicKey, sig *Signature) (err os.Error) { - h := sig.Hash.New() +// keySignatureHash returns a Hash of the message that needs to be signed for +// pk to assert a subkey relationship to signed. +func keySignatureHash(pk, signed *PublicKey, sig *Signature) (h hash.Hash, err os.Error) { + h = sig.Hash.New() if h == nil { - return error.UnsupportedError("hash function") + return nil, error.UnsupportedError("hash function") } // RFC 4880, section 5.2.4 pk.SerializeSignaturePrefix(h) - pk.Serialize(h) + pk.serializeWithoutHeaders(h) signed.SerializeSignaturePrefix(h) - signed.Serialize(h) + signed.serializeWithoutHeaders(h) + return +} +// VerifyKeySignature returns nil iff sig is a valid signature, made by this +// public key, of signed. +func (pk *PublicKey) VerifyKeySignature(signed *PublicKey, sig *Signature) (err os.Error) { + h, err := keySignatureHash(pk, signed, sig) + if err != nil { + return err + } return pk.VerifySignature(h, sig) } -// VerifyUserIdSignature returns nil iff sig is a valid signature, make by this -// public key, of the given user id. -func (pk *PublicKey) VerifyUserIdSignature(id string, sig *Signature) (err os.Error) { - h := sig.Hash.New() +// userIdSignatureHash returns a Hash of the message that needs to be signed +// to assert that pk is a valid key for id. +func userIdSignatureHash(id string, pk *PublicKey, sig *Signature) (h hash.Hash, err os.Error) { + h = sig.Hash.New() if h == nil { - return error.UnsupportedError("hash function") + return nil, error.UnsupportedError("hash function") } // RFC 4880, section 5.2.4 pk.SerializeSignaturePrefix(h) - pk.Serialize(h) + pk.serializeWithoutHeaders(h) var buf [5]byte buf[0] = 0xb4 @@ -249,6 +304,16 @@ func (pk *PublicKey) VerifyUserIdSignature(id string, sig *Signature) (err os.Er h.Write(buf[:]) h.Write([]byte(id)) + return +} + +// VerifyUserIdSignature returns nil iff sig is a valid signature, made by this +// public key, of id. +func (pk *PublicKey) VerifyUserIdSignature(id string, sig *Signature) (err os.Error) { + h, err := userIdSignatureHash(id, pk, sig) + if err != nil { + return err + } return pk.VerifySignature(h, sig) } @@ -272,7 +337,7 @@ type parsedMPI struct { bitLength uint16 } -// writeMPIs is a utility function for serialising several big integers to the +// writeMPIs is a utility function for serializing several big integers to the // given Writer. func writeMPIs(w io.Writer, mpis ...parsedMPI) (err os.Error) { for _, mpi := range mpis { diff --git a/src/pkg/crypto/openpgp/packet/public_key_test.go b/src/pkg/crypto/openpgp/packet/public_key_test.go index 069388c14..6e8bfbce6 100644 --- a/src/pkg/crypto/openpgp/packet/public_key_test.go +++ b/src/pkg/crypto/openpgp/packet/public_key_test.go @@ -28,12 +28,12 @@ func TestPublicKeyRead(t *testing.T) { packet, err := Read(readerFromHex(test.hexData)) if err != nil { t.Errorf("#%d: Read error: %s", i, err) - return + continue } pk, ok := packet.(*PublicKey) if !ok { t.Errorf("#%d: failed to parse, got: %#v", i, packet) - return + continue } if pk.PubKeyAlgo != test.pubKeyAlgo { t.Errorf("#%d: bad public key algorithm got:%x want:%x", i, pk.PubKeyAlgo, test.pubKeyAlgo) @@ -57,6 +57,38 @@ func TestPublicKeyRead(t *testing.T) { } } +func TestPublicKeySerialize(t *testing.T) { + for i, test := range pubKeyTests { + packet, err := Read(readerFromHex(test.hexData)) + if err != nil { + t.Errorf("#%d: Read error: %s", i, err) + continue + } + pk, ok := packet.(*PublicKey) + if !ok { + t.Errorf("#%d: failed to parse, got: %#v", i, packet) + continue + } + serializeBuf := bytes.NewBuffer(nil) + err = pk.Serialize(serializeBuf) + if err != nil { + t.Errorf("#%d: failed to serialize: %s", i, err) + continue + } + + packet, err = Read(serializeBuf) + if err != nil { + t.Errorf("#%d: Read error (from serialized data): %s", i, err) + continue + } + pk, ok = packet.(*PublicKey) + if !ok { + t.Errorf("#%d: failed to parse serialized data, got: %#v", i, packet) + continue + } + } +} + const rsaFingerprintHex = "5fb74b1d03b1e3cb31bc2f8aa34d7e18c20c31bb" const rsaPkDataHex = "988d044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd0011010001" diff --git a/src/pkg/crypto/openpgp/packet/signature.go b/src/pkg/crypto/openpgp/packet/signature.go index 719657e76..3169bac1e 100644 --- a/src/pkg/crypto/openpgp/packet/signature.go +++ b/src/pkg/crypto/openpgp/packet/signature.go @@ -393,7 +393,7 @@ func (sig *Signature) buildHashSuffix() (err os.Error) { sig.HashSuffix[3], ok = s2k.HashToHashId(sig.Hash) if !ok { sig.HashSuffix = nil - return error.InvalidArgumentError("hash cannot be repesented in OpenPGP: " + strconv.Itoa(int(sig.Hash))) + return error.InvalidArgumentError("hash cannot be represented in OpenPGP: " + strconv.Itoa(int(sig.Hash))) } sig.HashSuffix[4] = byte(hashedSubpacketsLen >> 8) sig.HashSuffix[5] = byte(hashedSubpacketsLen) @@ -420,28 +420,46 @@ func (sig *Signature) signPrepareHash(h hash.Hash) (digest []byte, err os.Error) return } -// SignRSA signs a message with an RSA private key. The hash, h, must contain +// Sign signs a message with a private key. The hash, h, must contain // the hash of the message to be signed and will be mutated by this function. // On success, the signature is stored in sig. Call Serialize to write it out. -func (sig *Signature) SignRSA(h hash.Hash, priv *rsa.PrivateKey) (err os.Error) { +func (sig *Signature) Sign(h hash.Hash, priv *PrivateKey) (err os.Error) { digest, err := sig.signPrepareHash(h) if err != nil { return } - sig.RSASignature, err = rsa.SignPKCS1v15(rand.Reader, priv, sig.Hash, digest) + + switch priv.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + sig.RSASignature, err = rsa.SignPKCS1v15(rand.Reader, priv.PrivateKey.(*rsa.PrivateKey), sig.Hash, digest) + case PubKeyAlgoDSA: + sig.DSASigR, sig.DSASigS, err = dsa.Sign(rand.Reader, priv.PrivateKey.(*dsa.PrivateKey), digest) + default: + err = error.UnsupportedError("public key algorithm: " + strconv.Itoa(int(sig.PubKeyAlgo))) + } + return } -// SignDSA signs a message with a DSA private key. The hash, h, must contain -// the hash of the message to be signed and will be mutated by this function. -// On success, the signature is stored in sig. Call Serialize to write it out. -func (sig *Signature) SignDSA(h hash.Hash, priv *dsa.PrivateKey) (err os.Error) { - digest, err := sig.signPrepareHash(h) +// SignUserId computes a signature from priv, asserting that pub is a valid +// key for the identity id. On success, the signature is stored in sig. Call +// Serialize to write it out. +func (sig *Signature) SignUserId(id string, pub *PublicKey, priv *PrivateKey) os.Error { + h, err := userIdSignatureHash(id, pub, sig) if err != nil { - return + return nil } - sig.DSASigR, sig.DSASigS, err = dsa.Sign(rand.Reader, priv, digest) - return + return sig.Sign(h, priv) +} + +// SignKey computes a signature from priv, asserting that pub is a subkey. On +// success, the signature is stored in sig. Call Serialize to write it out. +func (sig *Signature) SignKey(pub *PublicKey, priv *PrivateKey) os.Error { + h, err := keySignatureHash(&priv.PublicKey, pub, sig) + if err != nil { + return err + } + return sig.Sign(h, priv) } // Serialize marshals sig to w. SignRSA or SignDSA must have been called first. @@ -455,10 +473,8 @@ func (sig *Signature) Serialize(w io.Writer) (err os.Error) { case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: sigLength = len(sig.RSASignature) case PubKeyAlgoDSA: - sigLength = 2 /* MPI length */ - sigLength += (sig.DSASigR.BitLen() + 7) / 8 - sigLength += 2 /* MPI length */ - sigLength += (sig.DSASigS.BitLen() + 7) / 8 + sigLength = mpiLength(sig.DSASigR) + sigLength += mpiLength(sig.DSASigS) default: panic("impossible") } diff --git a/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted.go b/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted.go index d9010f88a..25d264acf 100644 --- a/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted.go +++ b/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted.go @@ -5,6 +5,7 @@ package packet import ( + "bytes" "crypto/cipher" "crypto/openpgp/error" "crypto/openpgp/s2k" @@ -27,6 +28,8 @@ type SymmetricKeyEncrypted struct { encryptedKey []byte } +const symmetricKeyEncryptedVersion = 4 + func (ske *SymmetricKeyEncrypted) parse(r io.Reader) (err os.Error) { // RFC 4880, section 5.3. var buf [2]byte @@ -34,7 +37,7 @@ func (ske *SymmetricKeyEncrypted) parse(r io.Reader) (err os.Error) { if err != nil { return } - if buf[0] != 4 { + if buf[0] != symmetricKeyEncryptedVersion { return error.UnsupportedError("SymmetricKeyEncrypted version") } ske.CipherFunc = CipherFunction(buf[1]) @@ -100,3 +103,60 @@ func (ske *SymmetricKeyEncrypted) Decrypt(passphrase []byte) os.Error { ske.Encrypted = false return nil } + +// SerializeSymmetricKeyEncrypted serializes a symmetric key packet to w. The +// packet contains a random session key, encrypted by a key derived from the +// given passphrase. The session key is returned and must be passed to +// SerializeSymmetricallyEncrypted. +func SerializeSymmetricKeyEncrypted(w io.Writer, rand io.Reader, passphrase []byte, cipherFunc CipherFunction) (key []byte, err os.Error) { + keySize := cipherFunc.keySize() + if keySize == 0 { + return nil, error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(cipherFunc))) + } + + s2kBuf := new(bytes.Buffer) + keyEncryptingKey := make([]byte, keySize) + // s2k.Serialize salts and stretches the passphrase, and writes the + // resulting key to keyEncryptingKey and the s2k descriptor to s2kBuf. + err = s2k.Serialize(s2kBuf, keyEncryptingKey, rand, passphrase) + if err != nil { + return + } + s2kBytes := s2kBuf.Bytes() + + packetLength := 2 /* header */ + len(s2kBytes) + 1 /* cipher type */ + keySize + err = serializeHeader(w, packetTypeSymmetricKeyEncrypted, packetLength) + if err != nil { + return + } + + var buf [2]byte + buf[0] = symmetricKeyEncryptedVersion + buf[1] = byte(cipherFunc) + _, err = w.Write(buf[:]) + if err != nil { + return + } + _, err = w.Write(s2kBytes) + if err != nil { + return + } + + sessionKey := make([]byte, keySize) + _, err = io.ReadFull(rand, sessionKey) + if err != nil { + return + } + iv := make([]byte, cipherFunc.blockSize()) + c := cipher.NewCFBEncrypter(cipherFunc.new(keyEncryptingKey), iv) + encryptedCipherAndKey := make([]byte, keySize+1) + c.XORKeyStream(encryptedCipherAndKey, buf[1:]) + c.XORKeyStream(encryptedCipherAndKey[1:], sessionKey) + _, err = w.Write(encryptedCipherAndKey) + if err != nil { + return + } + + key = sessionKey + return +} diff --git a/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted_test.go b/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted_test.go index 717c8ffa6..823ec400d 100644 --- a/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted_test.go +++ b/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted_test.go @@ -6,6 +6,7 @@ package packet import ( "bytes" + "crypto/rand" "encoding/hex" "io/ioutil" "os" @@ -60,3 +61,41 @@ func TestSymmetricKeyEncrypted(t *testing.T) { const symmetricallyEncryptedHex = "8c0d04030302371a0b38d884f02060c91cf97c9973b8e58e028e9501708ccfe618fb92afef7fa2d80ddadd93cf" const symmetricallyEncryptedContentsHex = "cb1062004d14c4df636f6e74656e74732e0a" + +func TestSerializeSymmetricKeyEncrypted(t *testing.T) { + buf := bytes.NewBuffer(nil) + passphrase := []byte("testing") + cipherFunc := CipherAES128 + + key, err := SerializeSymmetricKeyEncrypted(buf, rand.Reader, passphrase, cipherFunc) + if err != nil { + t.Errorf("failed to serialize: %s", err) + return + } + + p, err := Read(buf) + if err != nil { + t.Errorf("failed to reparse: %s", err) + return + } + ske, ok := p.(*SymmetricKeyEncrypted) + if !ok { + t.Errorf("parsed a different packet type: %#v", p) + return + } + + if !ske.Encrypted { + t.Errorf("SKE not encrypted but should be") + } + if ske.CipherFunc != cipherFunc { + t.Errorf("SKE cipher function is %d (expected %d)", ske.CipherFunc, cipherFunc) + } + err = ske.Decrypt(passphrase) + if err != nil { + t.Errorf("failed to decrypt reparsed SKE: %s", err) + return + } + if !bytes.Equal(key, ske.Key) { + t.Errorf("keys don't match after Decrpyt: %x (original) vs %x (parsed)", key, ske.Key) + } +} diff --git a/src/pkg/crypto/openpgp/packet/symmetrically_encrypted.go b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted.go index fc19ffe80..236c36774 100644 --- a/src/pkg/crypto/openpgp/packet/symmetrically_encrypted.go +++ b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted.go @@ -7,6 +7,7 @@ package packet import ( "crypto/cipher" "crypto/openpgp/error" + "crypto/rand" "crypto/sha1" "crypto/subtle" "hash" @@ -24,6 +25,8 @@ type SymmetricallyEncrypted struct { prefix []byte } +const symmetricallyEncryptedVersion = 1 + func (se *SymmetricallyEncrypted) parse(r io.Reader) os.Error { if se.MDC { // See RFC 4880, section 5.13. @@ -32,7 +35,7 @@ func (se *SymmetricallyEncrypted) parse(r io.Reader) os.Error { if err != nil { return err } - if buf[0] != 1 { + if buf[0] != symmetricallyEncryptedVersion { return error.UnsupportedError("unknown SymmetricallyEncrypted version") } } @@ -174,6 +177,9 @@ func (ser *seMDCReader) Read(buf []byte) (n int, err os.Error) { return } +// This is a new-format packet tag byte for a type 19 (MDC) packet. +const mdcPacketTagByte = byte(0x80) | 0x40 | 19 + func (ser *seMDCReader) Close() os.Error { if ser.error { return error.SignatureError("error during reading") @@ -191,16 +197,95 @@ func (ser *seMDCReader) Close() os.Error { } } - // This is a new-format packet tag byte for a type 19 (MDC) packet. - const mdcPacketTagByte = byte(0x80) | 0x40 | 19 if ser.trailer[0] != mdcPacketTagByte || ser.trailer[1] != sha1.Size { return error.SignatureError("MDC packet not found") } ser.h.Write(ser.trailer[:2]) final := ser.h.Sum() - if subtle.ConstantTimeCompare(final, ser.trailer[2:]) == 1 { + if subtle.ConstantTimeCompare(final, ser.trailer[2:]) != 1 { return error.SignatureError("hash mismatch") } return nil } + +// An seMDCWriter writes through to an io.WriteCloser while maintains a running +// hash of the data written. On close, it emits an MDC packet containing the +// running hash. +type seMDCWriter struct { + w io.WriteCloser + h hash.Hash +} + +func (w *seMDCWriter) Write(buf []byte) (n int, err os.Error) { + w.h.Write(buf) + return w.w.Write(buf) +} + +func (w *seMDCWriter) Close() (err os.Error) { + var buf [mdcTrailerSize]byte + + buf[0] = mdcPacketTagByte + buf[1] = sha1.Size + w.h.Write(buf[:2]) + digest := w.h.Sum() + copy(buf[2:], digest) + + _, err = w.w.Write(buf[:]) + if err != nil { + return + } + return w.w.Close() +} + +// noOpCloser is like an ioutil.NopCloser, but for an io.Writer. +type noOpCloser struct { + w io.Writer +} + +func (c noOpCloser) Write(data []byte) (n int, err os.Error) { + return c.w.Write(data) +} + +func (c noOpCloser) Close() os.Error { + return nil +} + +// SerializeSymmetricallyEncrypted serializes a symmetrically encrypted packet +// to w and returns a WriteCloser to which the to-be-encrypted packets can be +// written. +func SerializeSymmetricallyEncrypted(w io.Writer, c CipherFunction, key []byte) (contents io.WriteCloser, err os.Error) { + if c.keySize() != len(key) { + return nil, error.InvalidArgumentError("SymmetricallyEncrypted.Serialize: bad key length") + } + writeCloser := noOpCloser{w} + ciphertext, err := serializeStreamHeader(writeCloser, packetTypeSymmetricallyEncryptedMDC) + if err != nil { + return + } + + _, err = ciphertext.Write([]byte{symmetricallyEncryptedVersion}) + if err != nil { + return + } + + block := c.new(key) + blockSize := block.BlockSize() + iv := make([]byte, blockSize) + _, err = rand.Reader.Read(iv) + if err != nil { + return + } + s, prefix := cipher.NewOCFBEncrypter(block, iv, cipher.OCFBNoResync) + _, err = ciphertext.Write(prefix) + if err != nil { + return + } + plaintext := cipher.StreamWriter{S: s, W: ciphertext} + + h := sha1.New() + h.Write(iv) + h.Write(iv[blockSize-2:]) + contents = &seMDCWriter{w: plaintext, h: h} + return +} diff --git a/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go index 5543b2029..ba5606e6c 100644 --- a/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go +++ b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go @@ -9,6 +9,7 @@ import ( "crypto/openpgp/error" "crypto/sha1" "encoding/hex" + "io" "io/ioutil" "os" "testing" @@ -76,3 +77,48 @@ func testMDCReader(t *testing.T) { } const mdcPlaintextHex = "a302789c3b2d93c4e0eb9aba22283539b3203335af44a134afb800c849cb4c4de10200aff40b45d31432c80cb384299a0655966d6939dfdeed1dddf980" + +func TestSerialize(t *testing.T) { + buf := bytes.NewBuffer(nil) + c := CipherAES128 + key := make([]byte, c.keySize()) + + w, err := SerializeSymmetricallyEncrypted(buf, c, key) + if err != nil { + t.Errorf("error from SerializeSymmetricallyEncrypted: %s", err) + return + } + + contents := []byte("hello world\n") + + w.Write(contents) + w.Close() + + p, err := Read(buf) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + se, ok := p.(*SymmetricallyEncrypted) + if !ok { + t.Errorf("didn't read a *SymmetricallyEncrypted") + return + } + + r, err := se.Decrypt(c, key) + if err != nil { + t.Errorf("error from Decrypt: %s", err) + return + } + + contentsCopy := bytes.NewBuffer(nil) + _, err = io.Copy(contentsCopy, r) + if err != nil { + t.Errorf("error from io.Copy: %s", err) + return + } + if !bytes.Equal(contentsCopy.Bytes(), contents) { + t.Errorf("contents not equal got: %x want: %x", contentsCopy.Bytes(), contents) + } +} diff --git a/src/pkg/crypto/openpgp/packet/userid.go b/src/pkg/crypto/openpgp/packet/userid.go index ed2ad7774..0580ba3ed 100644 --- a/src/pkg/crypto/openpgp/packet/userid.go +++ b/src/pkg/crypto/openpgp/packet/userid.go @@ -20,6 +20,51 @@ type UserId struct { Name, Comment, Email string } +func hasInvalidCharacters(s string) bool { + for _, c := range s { + switch c { + case '(', ')', '<', '>', 0: + return true + } + } + return false +} + +// NewUserId returns a UserId or nil if any of the arguments contain invalid +// characters. The invalid characters are '\x00', '(', ')', '<' and '>' +func NewUserId(name, comment, email string) *UserId { + // RFC 4880 doesn't deal with the structure of userid strings; the + // name, comment and email form is just a convention. However, there's + // no convention about escaping the metacharacters and GPG just refuses + // to create user ids where, say, the name contains a '('. We mirror + // this behaviour. + + if hasInvalidCharacters(name) || hasInvalidCharacters(comment) || hasInvalidCharacters(email) { + return nil + } + + uid := new(UserId) + uid.Name, uid.Comment, uid.Email = name, comment, email + uid.Id = name + if len(comment) > 0 { + if len(uid.Id) > 0 { + uid.Id += " " + } + uid.Id += "(" + uid.Id += comment + uid.Id += ")" + } + if len(email) > 0 { + if len(uid.Id) > 0 { + uid.Id += " " + } + uid.Id += "<" + uid.Id += email + uid.Id += ">" + } + return uid +} + func (uid *UserId) parse(r io.Reader) (err os.Error) { // RFC 4880, section 5.11 b, err := ioutil.ReadAll(r) @@ -31,6 +76,17 @@ func (uid *UserId) parse(r io.Reader) (err os.Error) { return } +// Serialize marshals uid to w in the form of an OpenPGP packet, including +// header. +func (uid *UserId) Serialize(w io.Writer) os.Error { + err := serializeHeader(w, packetTypeUserId, len(uid.Id)) + if err != nil { + return err + } + _, err = w.Write([]byte(uid.Id)) + return err +} + // parseUserId extracts the name, comment and email from a user id string that // is formatted as "Full Name (Comment) <email@example.com>". func parseUserId(id string) (name, comment, email string) { diff --git a/src/pkg/crypto/openpgp/packet/userid_test.go b/src/pkg/crypto/openpgp/packet/userid_test.go index 394873dc3..296819389 100644 --- a/src/pkg/crypto/openpgp/packet/userid_test.go +++ b/src/pkg/crypto/openpgp/packet/userid_test.go @@ -40,3 +40,48 @@ func TestParseUserId(t *testing.T) { } } } + +var newUserIdTests = []struct { + name, comment, email, id string +}{ + {"foo", "", "", "foo"}, + {"", "bar", "", "(bar)"}, + {"", "", "baz", "<baz>"}, + {"foo", "bar", "", "foo (bar)"}, + {"foo", "", "baz", "foo <baz>"}, + {"", "bar", "baz", "(bar) <baz>"}, + {"foo", "bar", "baz", "foo (bar) <baz>"}, +} + +func TestNewUserId(t *testing.T) { + for i, test := range newUserIdTests { + uid := NewUserId(test.name, test.comment, test.email) + if uid == nil { + t.Errorf("#%d: returned nil", i) + continue + } + if uid.Id != test.id { + t.Errorf("#%d: got '%s', want '%s'", i, uid.Id, test.id) + } + } +} + +var invalidNewUserIdTests = []struct { + name, comment, email string +}{ + {"foo(", "", ""}, + {"foo<", "", ""}, + {"", "bar)", ""}, + {"", "bar<", ""}, + {"", "", "baz>"}, + {"", "", "baz)"}, + {"", "", "baz\x00"}, +} + +func TestNewUserIdWithInvalidInput(t *testing.T) { + for i, test := range invalidNewUserIdTests { + if uid := NewUserId(test.name, test.comment, test.email); uid != nil { + t.Errorf("#%d: returned non-nil value: %#v", i, uid) + } + } +} diff --git a/src/pkg/crypto/openpgp/read.go b/src/pkg/crypto/openpgp/read.go index 4f84dff82..46fcde363 100644 --- a/src/pkg/crypto/openpgp/read.go +++ b/src/pkg/crypto/openpgp/read.go @@ -44,7 +44,7 @@ type MessageDetails struct { DecryptedWith Key // the private key used to decrypt the message, if any. IsSigned bool // true if the message is signed. SignedByKeyId uint64 // the key id of the signer, if any. - SignedBy *Key // the key of the signer, if availible. + SignedBy *Key // the key of the signer, if available. LiteralData *packet.LiteralData // the metadata of the contents UnverifiedBody io.Reader // the contents of the message. @@ -145,7 +145,7 @@ ParsePackets: // function so that it can decrypt a key or give us a passphrase. FindKey: for { - // See if any of the keys already have a private key availible + // See if any of the keys already have a private key available candidates = candidates[:0] candidateFingerprints := make(map[string]bool) @@ -214,7 +214,7 @@ FindKey: return readSignedMessage(packets, md, keyring) } -// readSignedMessage reads a possibily signed message if mdin is non-zero then +// readSignedMessage reads a possibly signed message if mdin is non-zero then // that structure is updated and returned. Otherwise a fresh MessageDetails is // used. func readSignedMessage(packets *packet.Reader, mdin *MessageDetails, keyring KeyRing) (md *MessageDetails, err os.Error) { @@ -274,13 +274,13 @@ FindLiteralData: // hashForSignature returns a pair of hashes that can be used to verify a // signature. The signature may specify that the contents of the signed message -// should be preprocessed (i.e. to normalise line endings). Thus this function +// should be preprocessed (i.e. to normalize line endings). Thus this function // returns two hashes. The second should be used to hash the message itself and // performs any needed preprocessing. func hashForSignature(hashId crypto.Hash, sigType packet.SignatureType) (hash.Hash, hash.Hash, os.Error) { h := hashId.New() if h == nil { - return nil, nil, error.UnsupportedError("hash not availible: " + strconv.Itoa(int(hashId))) + return nil, nil, error.UnsupportedError("hash not available: " + strconv.Itoa(int(hashId))) } switch sigType { diff --git a/src/pkg/crypto/openpgp/read_test.go b/src/pkg/crypto/openpgp/read_test.go index 423c85b0f..a040b5b1b 100644 --- a/src/pkg/crypto/openpgp/read_test.go +++ b/src/pkg/crypto/openpgp/read_test.go @@ -193,9 +193,9 @@ func TestSymmetricallyEncrypted(t *testing.T) { t.Errorf("ReadAll: %s", err) } - expectedCreatationTime := uint32(1295992998) - if md.LiteralData.Time != expectedCreatationTime { - t.Errorf("LiteralData.Time is %d, want %d", md.LiteralData.Time, expectedCreatationTime) + expectedCreationTime := uint32(1295992998) + if md.LiteralData.Time != expectedCreationTime { + t.Errorf("LiteralData.Time is %d, want %d", md.LiteralData.Time, expectedCreationTime) } if string(contents) != expected { diff --git a/src/pkg/crypto/openpgp/s2k/s2k.go b/src/pkg/crypto/openpgp/s2k/s2k.go index 93b7582fa..da926a76e 100644 --- a/src/pkg/crypto/openpgp/s2k/s2k.go +++ b/src/pkg/crypto/openpgp/s2k/s2k.go @@ -90,7 +90,7 @@ func Parse(r io.Reader) (f func(out, in []byte), err os.Error) { } h := hash.New() if h == nil { - return nil, error.UnsupportedError("hash not availible: " + strconv.Itoa(int(hash))) + return nil, error.UnsupportedError("hash not available: " + strconv.Itoa(int(hash))) } switch buf[0] { @@ -123,6 +123,26 @@ func Parse(r io.Reader) (f func(out, in []byte), err os.Error) { return nil, error.UnsupportedError("S2K function") } +// Serialize salts and stretches the given passphrase and writes the resulting +// key into key. It also serializes an S2K descriptor to w. +func Serialize(w io.Writer, key []byte, rand io.Reader, passphrase []byte) os.Error { + var buf [11]byte + buf[0] = 3 /* iterated and salted */ + buf[1], _ = HashToHashId(crypto.SHA1) + salt := buf[2:10] + if _, err := io.ReadFull(rand, salt); err != nil { + return err + } + const count = 65536 // this is the default in gpg + buf[10] = 96 // 65536 iterations + if _, err := w.Write(buf[:]); err != nil { + return err + } + + Iterated(key, crypto.SHA1.New(), passphrase, salt, count) + return nil +} + // hashToHashIdMapping contains pairs relating OpenPGP's hash identifier with // Go's crypto.Hash type. See RFC 4880, section 9.4. var hashToHashIdMapping = []struct { diff --git a/src/pkg/crypto/openpgp/s2k/s2k_test.go b/src/pkg/crypto/openpgp/s2k/s2k_test.go index 75bc47ec1..27d2e9ae0 100644 --- a/src/pkg/crypto/openpgp/s2k/s2k_test.go +++ b/src/pkg/crypto/openpgp/s2k/s2k_test.go @@ -7,6 +7,7 @@ package s2k import ( "bytes" "crypto/sha1" + "crypto/rand" "encoding/hex" "testing" ) @@ -95,3 +96,26 @@ func TestParse(t *testing.T) { } } } + + +func TestSerialize(t *testing.T) { + buf := bytes.NewBuffer(nil) + key := make([]byte, 16) + passphrase := []byte("testing") + err := Serialize(buf, key, rand.Reader, passphrase) + if err != nil { + t.Errorf("failed to serialize: %s", err) + return + } + + f, err := Parse(buf) + if err != nil { + t.Errorf("failed to reparse: %s", err) + return + } + key2 := make([]byte, len(key)) + f(key2, passphrase) + if !bytes.Equal(key2, key) { + t.Errorf("keys don't match: %x (serialied) vs %x (parsed)", key, key2) + } +} diff --git a/src/pkg/crypto/openpgp/write.go b/src/pkg/crypto/openpgp/write.go index ef7b11230..48c86f604 100644 --- a/src/pkg/crypto/openpgp/write.go +++ b/src/pkg/crypto/openpgp/write.go @@ -6,15 +6,13 @@ package openpgp import ( "crypto" - "crypto/dsa" "crypto/openpgp/armor" "crypto/openpgp/error" "crypto/openpgp/packet" - "crypto/rsa" + "crypto/rand" _ "crypto/sha256" "io" "os" - "strconv" "time" ) @@ -77,20 +75,43 @@ func detachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.S } io.Copy(wrappedHash, message) - switch signer.PrivateKey.PubKeyAlgo { - case packet.PubKeyAlgoRSA, packet.PubKeyAlgoRSASignOnly: - priv := signer.PrivateKey.PrivateKey.(*rsa.PrivateKey) - err = sig.SignRSA(h, priv) - case packet.PubKeyAlgoDSA: - priv := signer.PrivateKey.PrivateKey.(*dsa.PrivateKey) - err = sig.SignDSA(h, priv) - default: - err = error.UnsupportedError("public key algorithm: " + strconv.Itoa(int(sig.PubKeyAlgo))) - } - + err = sig.Sign(h, signer.PrivateKey) if err != nil { return } return sig.Serialize(w) } + +// FileHints contains metadata about encrypted files. This metadata is, itself, +// encrypted. +type FileHints struct { + // IsBinary can be set to hint that the contents are binary data. + IsBinary bool + // FileName hints at the name of the file that should be written. It's + // truncated to 255 bytes if longer. It may be empty to suggest that the + // file should not be written to disk. It may be equal to "_CONSOLE" to + // suggest the data should not be written to disk. + FileName string + // EpochSeconds contains the modification time of the file, or 0 if not applicable. + EpochSeconds uint32 +} + +// SymmetricallyEncrypt acts like gpg -c: it encrypts a file with a passphrase. +// The resulting WriteCloser MUST be closed after the contents of the file have +// been written. +func SymmetricallyEncrypt(ciphertext io.Writer, passphrase []byte, hints *FileHints) (plaintext io.WriteCloser, err os.Error) { + if hints == nil { + hints = &FileHints{} + } + + key, err := packet.SerializeSymmetricKeyEncrypted(ciphertext, rand.Reader, passphrase, packet.CipherAES128) + if err != nil { + return + } + w, err := packet.SerializeSymmetricallyEncrypted(ciphertext, packet.CipherAES128, key) + if err != nil { + return + } + return packet.SerializeLiteral(w, hints.IsBinary, hints.FileName, hints.EpochSeconds) +} diff --git a/src/pkg/crypto/openpgp/write_test.go b/src/pkg/crypto/openpgp/write_test.go index 42cd0d27f..8551aeb63 100644 --- a/src/pkg/crypto/openpgp/write_test.go +++ b/src/pkg/crypto/openpgp/write_test.go @@ -6,7 +6,11 @@ package openpgp import ( "bytes" + "crypto/rand" + "os" + "io" "testing" + "time" ) func TestSignDetached(t *testing.T) { @@ -44,3 +48,75 @@ func TestSignDetachedDSA(t *testing.T) { testDetachedSignature(t, kring, out, signedInput, "check", testKey3KeyId) } + +func TestNewEntity(t *testing.T) { + if testing.Short() { + return + } + + e, err := NewEntity(rand.Reader, time.Seconds(), "Test User", "test", "test@example.com") + if err != nil { + t.Errorf("failed to create entity: %s", err) + return + } + + w := bytes.NewBuffer(nil) + if err := e.SerializePrivate(w); err != nil { + t.Errorf("failed to serialize entity: %s", err) + return + } + serialized := w.Bytes() + + el, err := ReadKeyRing(w) + if err != nil { + t.Errorf("failed to reparse entity: %s", err) + return + } + + if len(el) != 1 { + t.Errorf("wrong number of entities found, got %d, want 1", len(el)) + } + + w = bytes.NewBuffer(nil) + if err := e.SerializePrivate(w); err != nil { + t.Errorf("failed to serialize entity second time: %s", err) + return + } + + if !bytes.Equal(w.Bytes(), serialized) { + t.Errorf("results differed") + } +} + +func TestSymmetricEncryption(t *testing.T) { + buf := new(bytes.Buffer) + plaintext, err := SymmetricallyEncrypt(buf, []byte("testing"), nil) + if err != nil { + t.Errorf("error writing headers: %s", err) + return + } + message := []byte("hello world\n") + _, err = plaintext.Write(message) + if err != nil { + t.Errorf("error writing to plaintext writer: %s", err) + } + err = plaintext.Close() + if err != nil { + t.Errorf("error closing plaintext writer: %s", err) + } + + md, err := ReadMessage(buf, nil, func(keys []Key, symmetric bool) ([]byte, os.Error) { + return []byte("testing"), nil + }) + if err != nil { + t.Errorf("error rereading message: %s", err) + } + messageBuf := bytes.NewBuffer(nil) + _, err = io.Copy(messageBuf, md.UnverifiedBody) + if err != nil { + t.Errorf("error rereading message: %s", err) + } + if !bytes.Equal(message, messageBuf.Bytes()) { + t.Errorf("recovered message incorrect got '%s', want '%s'", messageBuf.Bytes(), message) + } +} diff --git a/src/pkg/crypto/rand/Makefile b/src/pkg/crypto/rand/Makefile index 88b6d71e3..d1321297d 100644 --- a/src/pkg/crypto/rand/Makefile +++ b/src/pkg/crypto/rand/Makefile @@ -8,6 +8,7 @@ TARG=crypto/rand GOFILES=\ rand.go\ + util.go\ GOFILES_freebsd=\ rand_unix.go\ diff --git a/src/pkg/crypto/rand/util.go b/src/pkg/crypto/rand/util.go new file mode 100644 index 000000000..77028476e --- /dev/null +++ b/src/pkg/crypto/rand/util.go @@ -0,0 +1,80 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rand + +import ( + "big" + "io" + "os" +) + +// Prime returns a number, p, of the given size, such that p is prime +// with high probability. +func Prime(rand io.Reader, bits int) (p *big.Int, err os.Error) { + if bits < 1 { + err = os.EINVAL + } + + b := uint(bits % 8) + if b == 0 { + b = 8 + } + + bytes := make([]byte, (bits+7)/8) + p = new(big.Int) + + for { + _, err = io.ReadFull(rand, bytes) + if err != nil { + return nil, err + } + + // Clear bits in the first byte to make sure the candidate has a size <= bits. + bytes[0] &= uint8(int(1<<b) - 1) + // Don't let the value be too small, i.e, set the most significant bit. + bytes[0] |= 1 << (b - 1) + // Make the value odd since an even number this large certainly isn't prime. + bytes[len(bytes)-1] |= 1 + + p.SetBytes(bytes) + if big.ProbablyPrime(p, 20) { + return + } + } + + return +} + +// Int returns a uniform random value in [0, max). +func Int(rand io.Reader, max *big.Int) (n *big.Int, err os.Error) { + k := (max.BitLen() + 7) / 8 + + // b is the number of bits in the most significant byte of max. + b := uint(max.BitLen() % 8) + if b == 0 { + b = 8 + } + + bytes := make([]byte, k) + n = new(big.Int) + + for { + _, err = io.ReadFull(rand, bytes) + if err != nil { + return nil, err + } + + // Clear bits in the first byte to increase the probability + // that the candidate is < max. + bytes[0] &= uint8(int(1<<b) - 1) + + n.SetBytes(bytes) + if n.Cmp(max) < 0 { + return + } + } + + return +} diff --git a/src/pkg/crypto/rsa/rsa.go b/src/pkg/crypto/rsa/rsa.go index e1813dbf9..380f71570 100644 --- a/src/pkg/crypto/rsa/rsa.go +++ b/src/pkg/crypto/rsa/rsa.go @@ -9,6 +9,7 @@ package rsa import ( "big" + "crypto/rand" "crypto/subtle" "hash" "io" @@ -18,69 +19,6 @@ import ( var bigZero = big.NewInt(0) var bigOne = big.NewInt(1) -// randomPrime returns a number, p, of the given size, such that p is prime -// with high probability. -func randomPrime(rand io.Reader, bits int) (p *big.Int, err os.Error) { - if bits < 1 { - err = os.EINVAL - } - - bytes := make([]byte, (bits+7)/8) - p = new(big.Int) - - for { - _, err = io.ReadFull(rand, bytes) - if err != nil { - return - } - - // Don't let the value be too small. - bytes[0] |= 0x80 - // Make the value odd since an even number this large certainly isn't prime. - bytes[len(bytes)-1] |= 1 - - p.SetBytes(bytes) - if big.ProbablyPrime(p, 20) { - return - } - } - - return -} - -// randomNumber returns a uniform random value in [0, max). -func randomNumber(rand io.Reader, max *big.Int) (n *big.Int, err os.Error) { - k := (max.BitLen() + 7) / 8 - - // r is the number of bits in the used in the most significant byte of - // max. - r := uint(max.BitLen() % 8) - if r == 0 { - r = 8 - } - - bytes := make([]byte, k) - n = new(big.Int) - - for { - _, err = io.ReadFull(rand, bytes) - if err != nil { - return - } - - // Clear bits in the first byte to increase the probability - // that the candidate is < max. - bytes[0] &= uint8(int(1<<r) - 1) - - n.SetBytes(bytes) - if n.Cmp(max) < 0 { - return - } - } - - return -} - // A PublicKey represents the public part of an RSA key. type PublicKey struct { N *big.Int // modulus @@ -94,7 +32,7 @@ type PrivateKey struct { Primes []*big.Int // prime factors of N, has >= 2 elements. // Precomputed contains precomputed values that speed up private - // operations, if availible. + // operations, if available. Precomputed PrecomputedValues } @@ -126,7 +64,7 @@ func (priv *PrivateKey) Validate() os.Error { // easy for an attack to generate composites that pass this test. for _, prime := range priv.Primes { if !big.ProbablyPrime(prime, 20) { - return os.ErrorString("Prime factor is composite") + return os.ErrorString("prime factor is composite") } } @@ -162,8 +100,8 @@ func (priv *PrivateKey) Validate() os.Error { } // GenerateKey generates an RSA keypair of the given bit size. -func GenerateKey(rand io.Reader, bits int) (priv *PrivateKey, err os.Error) { - return GenerateMultiPrimeKey(rand, 2, bits) +func GenerateKey(random io.Reader, bits int) (priv *PrivateKey, err os.Error) { + return GenerateMultiPrimeKey(random, 2, bits) } // GenerateMultiPrimeKey generates a multi-prime RSA keypair of the given bit @@ -176,7 +114,7 @@ func GenerateKey(rand io.Reader, bits int) (priv *PrivateKey, err os.Error) { // // [1] US patent 4405829 (1972, expired) // [2] http://www.cacr.math.uwaterloo.ca/techreports/2006/cacr2006-16.pdf -func GenerateMultiPrimeKey(rand io.Reader, nprimes int, bits int) (priv *PrivateKey, err os.Error) { +func GenerateMultiPrimeKey(random io.Reader, nprimes int, bits int) (priv *PrivateKey, err os.Error) { priv = new(PrivateKey) // Smaller public exponents lead to faster public key // operations. Since the exponent must be coprime to @@ -198,7 +136,7 @@ NextSetOfPrimes: for { todo := bits for i := 0; i < nprimes; i++ { - primes[i], err = randomPrime(rand, todo/(nprimes-i)) + primes[i], err = rand.Prime(random, todo/(nprimes-i)) if err != nil { return nil, err } @@ -293,7 +231,7 @@ func encrypt(c *big.Int, pub *PublicKey, m *big.Int) *big.Int { // EncryptOAEP encrypts the given message with RSA-OAEP. // The message must be no longer than the length of the public modulus less // twice the hash length plus 2. -func EncryptOAEP(hash hash.Hash, rand io.Reader, pub *PublicKey, msg []byte, label []byte) (out []byte, err os.Error) { +func EncryptOAEP(hash hash.Hash, random io.Reader, pub *PublicKey, msg []byte, label []byte) (out []byte, err os.Error) { hash.Reset() k := (pub.N.BitLen() + 7) / 8 if len(msg) > k-2*hash.Size()-2 { @@ -313,7 +251,7 @@ func EncryptOAEP(hash hash.Hash, rand io.Reader, pub *PublicKey, msg []byte, lab db[len(db)-len(msg)-1] = 1 copy(db[len(db)-len(msg):], msg) - _, err = io.ReadFull(rand, seed) + _, err = io.ReadFull(random, seed) if err != nil { return } @@ -405,7 +343,7 @@ func (priv *PrivateKey) Precompute() { // decrypt performs an RSA decryption, resulting in a plaintext integer. If a // random source is given, RSA blinding is used. -func decrypt(rand io.Reader, priv *PrivateKey, c *big.Int) (m *big.Int, err os.Error) { +func decrypt(random io.Reader, priv *PrivateKey, c *big.Int) (m *big.Int, err os.Error) { // TODO(agl): can we get away with reusing blinds? if c.Cmp(priv.N) > 0 { err = DecryptionError{} @@ -413,16 +351,16 @@ func decrypt(rand io.Reader, priv *PrivateKey, c *big.Int) (m *big.Int, err os.E } var ir *big.Int - if rand != nil { + if random != nil { // Blinding enabled. Blinding involves multiplying c by r^e. // Then the decryption operation performs (m^e * r^e)^d mod n // which equals mr mod n. The factor of r can then be removed - // by multipling by the multiplicative inverse of r. + // by multiplying by the multiplicative inverse of r. var r *big.Int for { - r, err = randomNumber(rand, priv.N) + r, err = rand.Int(random, priv.N) if err != nil { return } @@ -483,7 +421,7 @@ func decrypt(rand io.Reader, priv *PrivateKey, c *big.Int) (m *big.Int, err os.E // DecryptOAEP decrypts ciphertext using RSA-OAEP. // If rand != nil, DecryptOAEP uses RSA blinding to avoid timing side-channel attacks. -func DecryptOAEP(hash hash.Hash, rand io.Reader, priv *PrivateKey, ciphertext []byte, label []byte) (msg []byte, err os.Error) { +func DecryptOAEP(hash hash.Hash, random io.Reader, priv *PrivateKey, ciphertext []byte, label []byte) (msg []byte, err os.Error) { k := (priv.N.BitLen() + 7) / 8 if len(ciphertext) > k || k < hash.Size()*2+2 { @@ -493,7 +431,7 @@ func DecryptOAEP(hash hash.Hash, rand io.Reader, priv *PrivateKey, ciphertext [] c := new(big.Int).SetBytes(ciphertext) - m, err := decrypt(rand, priv, c) + m, err := decrypt(random, priv, c) if err != nil { return } diff --git a/src/pkg/crypto/subtle/constant_time_test.go b/src/pkg/crypto/subtle/constant_time_test.go index b28b73581..adab8e2e8 100644 --- a/src/pkg/crypto/subtle/constant_time_test.go +++ b/src/pkg/crypto/subtle/constant_time_test.go @@ -14,14 +14,14 @@ type TestConstantTimeCompareStruct struct { out int } -var testConstandTimeCompareData = []TestConstantTimeCompareStruct{ +var testConstantTimeCompareData = []TestConstantTimeCompareStruct{ {[]byte{}, []byte{}, 1}, {[]byte{0x11}, []byte{0x11}, 1}, {[]byte{0x12}, []byte{0x11}, 0}, } func TestConstantTimeCompare(t *testing.T) { - for i, test := range testConstandTimeCompareData { + for i, test := range testConstantTimeCompareData { if r := ConstantTimeCompare(test.a, test.b); r != test.out { t.Errorf("#%d bad result (got %x, want %x)", i, r, test.out) } diff --git a/src/pkg/crypto/tls/common.go b/src/pkg/crypto/tls/common.go index 204d25531..3efac9c13 100644 --- a/src/pkg/crypto/tls/common.go +++ b/src/pkg/crypto/tls/common.go @@ -87,7 +87,7 @@ const ( certTypeRSASign = 1 // A certificate containing an RSA key certTypeDSSSign = 2 // A certificate containing a DSA key certTypeRSAFixedDH = 3 // A certificate containing a static DH key - certTypeDSSFixedDH = 4 // A certficiate containing a static DH key + certTypeDSSFixedDH = 4 // A certificate containing a static DH key // Rest of these are reserved by the TLS spec ) @@ -100,6 +100,8 @@ type ConnectionState struct { // the certificate chain that was presented by the other side PeerCertificates []*x509.Certificate + // the verified certificate chains built from PeerCertificates. + VerifiedChains [][]*x509.Certificate } // A Config structure is used to configure a TLS client or server. After one diff --git a/src/pkg/crypto/tls/conn.go b/src/pkg/crypto/tls/conn.go index 63d56310c..097e182bd 100644 --- a/src/pkg/crypto/tls/conn.go +++ b/src/pkg/crypto/tls/conn.go @@ -34,7 +34,7 @@ type Conn struct { cipherSuite uint16 ocspResponse []byte // stapled OCSP response peerCertificates []*x509.Certificate - // verifedChains contains the certificate chains that we built, as + // verifiedChains contains the certificate chains that we built, as // opposed to the ones presented by the server. verifiedChains [][]*x509.Certificate @@ -237,7 +237,7 @@ func (hc *halfConn) decrypt(b *block) (bool, alert) { // "Password Interception in a SSL/TLS Channel", Brice // Canvel et al. // - // However, our behaviour matches OpenSSL, so we leak + // However, our behavior matches OpenSSL, so we leak // only as much as they do. default: panic("unknown cipher type") @@ -410,7 +410,7 @@ func (hc *halfConn) freeBlock(b *block) { // splitBlock splits a block after the first n bytes, // returning a block with those n bytes and a -// block with the remaindec. the latter may be nil. +// block with the remainder. the latter may be nil. func (hc *halfConn) splitBlock(b *block, n int) (*block, *block) { if len(b.data) <= n { return b, nil @@ -768,6 +768,7 @@ func (c *Conn) ConnectionState() ConnectionState { state.NegotiatedProtocolIsMutual = !c.clientProtocolFallback state.CipherSuite = c.cipherSuite state.PeerCertificates = c.peerCertificates + state.VerifiedChains = c.verifiedChains } return state diff --git a/src/pkg/crypto/tls/generate_cert.go b/src/pkg/crypto/tls/generate_cert.go index 5b8c700e5..f46188879 100644 --- a/src/pkg/crypto/tls/generate_cert.go +++ b/src/pkg/crypto/tls/generate_cert.go @@ -59,7 +59,7 @@ func main() { certOut.Close() log.Print("written cert.pem\n") - keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0600) + keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { log.Print("failed to open key.pem for writing:", err) return diff --git a/src/pkg/crypto/tls/handshake_server.go b/src/pkg/crypto/tls/handshake_server.go index 37c8d154a..e9431c6fa 100644 --- a/src/pkg/crypto/tls/handshake_server.go +++ b/src/pkg/crypto/tls/handshake_server.go @@ -209,10 +209,10 @@ FindCipherSuite: // If we received a client cert in response to our certificate request message, // the client will send us a certificateVerifyMsg immediately after the - // clientKeyExchangeMsg. This message is a MD5SHA1 digest of all preceeding + // clientKeyExchangeMsg. This message is a MD5SHA1 digest of all preceding // handshake-layer messages that is signed using the private key corresponding // to the client's certificate. This allows us to verify that the client is in - // posession of the private key of the certificate. + // possession of the private key of the certificate. if len(c.peerCertificates) > 0 { msg, err = c.readHandshake() if err != nil { diff --git a/src/pkg/crypto/tls/key_agreement.go b/src/pkg/crypto/tls/key_agreement.go index 8edbb1190..84f90c45a 100644 --- a/src/pkg/crypto/tls/key_agreement.go +++ b/src/pkg/crypto/tls/key_agreement.go @@ -236,12 +236,12 @@ func (ka *ecdheRSAKeyAgreement) generateClientKeyExchange(config *Config, client xBytes := x.Bytes() copy(preMasterSecret[len(preMasterSecret)-len(xBytes):], xBytes) - serialised := ka.curve.Marshal(mx, my) + serialized := ka.curve.Marshal(mx, my) ckx := new(clientKeyExchangeMsg) - ckx.ciphertext = make([]byte, 1+len(serialised)) - ckx.ciphertext[0] = byte(len(serialised)) - copy(ckx.ciphertext[1:], serialised) + ckx.ciphertext = make([]byte, 1+len(serialized)) + ckx.ciphertext[0] = byte(len(serialized)) + copy(ckx.ciphertext[1:], serialized) return preMasterSecret, ckx, nil } diff --git a/src/pkg/crypto/tls/tls.go b/src/pkg/crypto/tls/tls.go index 7d0bb9f34..9e5c9270a 100644 --- a/src/pkg/crypto/tls/tls.go +++ b/src/pkg/crypto/tls/tls.go @@ -159,7 +159,7 @@ func X509KeyPair(certPEMBlock, keyPEMBlock []byte) (cert Certificate, err os.Err key, err := x509.ParsePKCS1PrivateKey(keyDERBlock.Bytes) if err != nil { - err = os.ErrorString("crypto/tls: failed to parse key") + err = os.ErrorString("crypto/tls: failed to parse key: " + err.String()) return } diff --git a/src/pkg/crypto/twofish/twofish.go b/src/pkg/crypto/twofish/twofish.go index 9303f03ff..1a1aac9b9 100644 --- a/src/pkg/crypto/twofish/twofish.go +++ b/src/pkg/crypto/twofish/twofish.go @@ -116,7 +116,7 @@ func (c *Cipher) Reset() { c.k[i] = 0 } for i := range c.s { - for j := 0; j < 265; j++ { + for j := 0; j < 256; j++ { c.s[i][j] = 0 } } diff --git a/src/pkg/crypto/x509/cert_pool.go b/src/pkg/crypto/x509/cert_pool.go index c295fd97e..16cd92efc 100644 --- a/src/pkg/crypto/x509/cert_pool.go +++ b/src/pkg/crypto/x509/cert_pool.go @@ -5,6 +5,7 @@ package x509 import ( + "crypto/x509/pkix" "encoding/pem" "strings" ) @@ -25,7 +26,7 @@ func NewCertPool() *CertPool { } } -func nameToKey(name *Name) string { +func nameToKey(name *pkix.Name) string { return strings.Join(name.Country, ",") + "/" + strings.Join(name.Organization, ",") + "/" + strings.Join(name.OrganizationalUnit, ",") + "/" + name.CommonName } diff --git a/src/pkg/crypto/x509/pkix/Makefile b/src/pkg/crypto/x509/pkix/Makefile new file mode 100644 index 000000000..e29b74c01 --- /dev/null +++ b/src/pkg/crypto/x509/pkix/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/x509/pkix +GOFILES=\ + pkix.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/x509/pkix/pkix.go b/src/pkg/crypto/x509/pkix/pkix.go new file mode 100644 index 000000000..7806b2a2e --- /dev/null +++ b/src/pkg/crypto/x509/pkix/pkix.go @@ -0,0 +1,167 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package pkix contains shared, low level structures used for ASN.1 parsing +// and serialization of X.509 certificates, CRL and OCSP. +package pkix + +import ( + "asn1" + "big" + "time" +) + +// AlgorithmIdentifier represents the ASN.1 structure of the same name. See RFC +// 5280, section 4.1.1.2. +type AlgorithmIdentifier struct { + Algorithm asn1.ObjectIdentifier + Parameters asn1.RawValue "optional" +} + +type RDNSequence []RelativeDistinguishedNameSET + +type RelativeDistinguishedNameSET []AttributeTypeAndValue + +type AttributeTypeAndValue struct { + Type asn1.ObjectIdentifier + Value interface{} +} + +// Extension represents the ASN.1 structure of the same name. See RFC +// 5280, section 4.2. +type Extension struct { + Id asn1.ObjectIdentifier + Critical bool "optional" + Value []byte +} + +// Name represents an X.509 distinguished name. This only includes the common +// elements of a DN. Additional elements in the name are ignored. +type Name struct { + Country, Organization, OrganizationalUnit []string + Locality, Province []string + StreetAddress, PostalCode []string + SerialNumber, CommonName string +} + +func (n *Name) FillFromRDNSequence(rdns *RDNSequence) { + for _, rdn := range *rdns { + if len(rdn) == 0 { + continue + } + atv := rdn[0] + value, ok := atv.Value.(string) + if !ok { + continue + } + + t := atv.Type + if len(t) == 4 && t[0] == 2 && t[1] == 5 && t[2] == 4 { + switch t[3] { + case 3: + n.CommonName = value + case 5: + n.SerialNumber = value + case 6: + n.Country = append(n.Country, value) + case 7: + n.Locality = append(n.Locality, value) + case 8: + n.Province = append(n.Province, value) + case 9: + n.StreetAddress = append(n.StreetAddress, value) + case 10: + n.Organization = append(n.Organization, value) + case 11: + n.OrganizationalUnit = append(n.OrganizationalUnit, value) + case 17: + n.PostalCode = append(n.PostalCode, value) + } + } + } +} + +var ( + oidCountry = []int{2, 5, 4, 6} + oidOrganization = []int{2, 5, 4, 10} + oidOrganizationalUnit = []int{2, 5, 4, 11} + oidCommonName = []int{2, 5, 4, 3} + oidSerialNumber = []int{2, 5, 4, 5} + oidLocality = []int{2, 5, 4, 7} + oidProvince = []int{2, 5, 4, 8} + oidStreetAddress = []int{2, 5, 4, 9} + oidPostalCode = []int{2, 5, 4, 17} +) + +// appendRDNs appends a relativeDistinguishedNameSET to the given RDNSequence +// and returns the new value. The relativeDistinguishedNameSET contains an +// attributeTypeAndValue for each of the given values. See RFC 5280, A.1, and +// search for AttributeTypeAndValue. +func appendRDNs(in RDNSequence, values []string, oid asn1.ObjectIdentifier) RDNSequence { + if len(values) == 0 { + return in + } + + s := make([]AttributeTypeAndValue, len(values)) + for i, value := range values { + s[i].Type = oid + s[i].Value = value + } + + return append(in, s) +} + +func (n Name) ToRDNSequence() (ret RDNSequence) { + ret = appendRDNs(ret, n.Country, oidCountry) + ret = appendRDNs(ret, n.Organization, oidOrganization) + ret = appendRDNs(ret, n.OrganizationalUnit, oidOrganizationalUnit) + ret = appendRDNs(ret, n.Locality, oidLocality) + ret = appendRDNs(ret, n.Province, oidProvince) + ret = appendRDNs(ret, n.StreetAddress, oidStreetAddress) + ret = appendRDNs(ret, n.PostalCode, oidPostalCode) + if len(n.CommonName) > 0 { + ret = appendRDNs(ret, []string{n.CommonName}, oidCommonName) + } + if len(n.SerialNumber) > 0 { + ret = appendRDNs(ret, []string{n.SerialNumber}, oidSerialNumber) + } + + return ret +} + +// CertificateList represents the ASN.1 structure of the same name. See RFC +// 5280, section 5.1. Use Certificate.CheckCRLSignature to verify the +// signature. +type CertificateList struct { + TBSCertList TBSCertificateList + SignatureAlgorithm AlgorithmIdentifier + SignatureValue asn1.BitString +} + +// HasExpired returns true iff currentTimeSeconds is past the expiry time of +// certList. +func (certList *CertificateList) HasExpired(currentTimeSeconds int64) bool { + return certList.TBSCertList.NextUpdate.Seconds() <= currentTimeSeconds +} + +// TBSCertificateList represents the ASN.1 structure of the same name. See RFC +// 5280, section 5.1. +type TBSCertificateList struct { + Raw asn1.RawContent + Version int "optional,default:2" + Signature AlgorithmIdentifier + Issuer RDNSequence + ThisUpdate *time.Time + NextUpdate *time.Time + RevokedCertificates []RevokedCertificate "optional" + Extensions []Extension "tag:0,optional,explicit" +} + +// RevokedCertificate represents the ASN.1 structure of the same name. See RFC +// 5280, section 5.1. +type RevokedCertificate struct { + SerialNumber *big.Int + RevocationTime *time.Time + Extensions []Extension "optional" +} diff --git a/src/pkg/crypto/x509/x509.go b/src/pkg/crypto/x509/x509.go index f2a039b5a..b10ffb0a2 100644 --- a/src/pkg/crypto/x509/x509.go +++ b/src/pkg/crypto/x509/x509.go @@ -11,9 +11,11 @@ import ( "bytes" "container/vector" "crypto" + "crypto/dsa" "crypto/rsa" "crypto/sha1" - "hash" + "crypto/x509/pkix" + "encoding/pem" "io" "os" "time" @@ -22,30 +24,25 @@ import ( // pkcs1PrivateKey is a structure which mirrors the PKCS#1 ASN.1 for an RSA private key. type pkcs1PrivateKey struct { Version int - N asn1.RawValue + N *big.Int E int - D asn1.RawValue - P asn1.RawValue - Q asn1.RawValue + D *big.Int + P *big.Int + Q *big.Int // We ignore these values, if present, because rsa will calculate them. - Dp asn1.RawValue "optional" - Dq asn1.RawValue "optional" - Qinv asn1.RawValue "optional" + Dp *big.Int "optional" + Dq *big.Int "optional" + Qinv *big.Int "optional" - AdditionalPrimes []pkcs1AddtionalRSAPrime "optional" + AdditionalPrimes []pkcs1AdditionalRSAPrime "optional" } -type pkcs1AddtionalRSAPrime struct { - Prime asn1.RawValue +type pkcs1AdditionalRSAPrime struct { + Prime *big.Int // We ignore these values because rsa will calculate them. - Exp asn1.RawValue - Coeff asn1.RawValue -} - -// rawValueIsInteger returns true iff the given ASN.1 RawValue is an INTEGER type. -func rawValueIsInteger(raw *asn1.RawValue) bool { - return raw.Class == 0 && raw.Tag == 2 && raw.IsCompound == false + Exp *big.Int + Coeff *big.Int } // ParsePKCS1PrivateKey returns an RSA private key from its ASN.1 PKCS#1 DER encoded form. @@ -64,29 +61,25 @@ func ParsePKCS1PrivateKey(der []byte) (key *rsa.PrivateKey, err os.Error) { return nil, os.ErrorString("x509: unsupported private key version") } - if !rawValueIsInteger(&priv.N) || - !rawValueIsInteger(&priv.D) || - !rawValueIsInteger(&priv.P) || - !rawValueIsInteger(&priv.Q) { - err = asn1.StructuralError{"tags don't match"} - return + if priv.N.Sign() <= 0 || priv.D.Sign() <= 0 || priv.P.Sign() <= 0 || priv.Q.Sign() <= 0 { + return nil, os.ErrorString("private key contains zero or negative value") } key = new(rsa.PrivateKey) key.PublicKey = rsa.PublicKey{ E: priv.E, - N: new(big.Int).SetBytes(priv.N.Bytes), + N: priv.N, } - key.D = new(big.Int).SetBytes(priv.D.Bytes) + key.D = priv.D key.Primes = make([]*big.Int, 2+len(priv.AdditionalPrimes)) - key.Primes[0] = new(big.Int).SetBytes(priv.P.Bytes) - key.Primes[1] = new(big.Int).SetBytes(priv.Q.Bytes) + key.Primes[0] = priv.P + key.Primes[1] = priv.Q for i, a := range priv.AdditionalPrimes { - if !rawValueIsInteger(&a.Prime) { - return nil, asn1.StructuralError{"tags don't match"} + if a.Prime.Sign() <= 0 { + return nil, os.ErrorString("private key contains zero or negative prime") } - key.Primes[i+2] = new(big.Int).SetBytes(a.Prime.Bytes) + key.Primes[i+2] = a.Prime // We ignore the other two values because rsa will calculate // them as needed. } @@ -100,19 +93,6 @@ func ParsePKCS1PrivateKey(der []byte) (key *rsa.PrivateKey, err os.Error) { return } -// rawValueForBig returns an asn1.RawValue which represents the given integer. -func rawValueForBig(n *big.Int) asn1.RawValue { - b := n.Bytes() - if n.Sign() >= 0 && len(b) > 0 && b[0]&0x80 != 0 { - // This positive number would be interpreted as a negative - // number in ASN.1 because the MSB is set. - padded := make([]byte, len(b)+1) - copy(padded[1:], b) - b = padded - } - return asn1.RawValue{Tag: 2, Bytes: b} -} - // MarshalPKCS1PrivateKey converts a private key to ASN.1 DER encoded form. func MarshalPKCS1PrivateKey(key *rsa.PrivateKey) []byte { key.Precompute() @@ -124,21 +104,21 @@ func MarshalPKCS1PrivateKey(key *rsa.PrivateKey) []byte { priv := pkcs1PrivateKey{ Version: version, - N: rawValueForBig(key.N), + N: key.N, E: key.PublicKey.E, - D: rawValueForBig(key.D), - P: rawValueForBig(key.Primes[0]), - Q: rawValueForBig(key.Primes[1]), - Dp: rawValueForBig(key.Precomputed.Dp), - Dq: rawValueForBig(key.Precomputed.Dq), - Qinv: rawValueForBig(key.Precomputed.Qinv), + D: key.D, + P: key.Primes[0], + Q: key.Primes[1], + Dp: key.Precomputed.Dp, + Dq: key.Precomputed.Dq, + Qinv: key.Precomputed.Qinv, } - priv.AdditionalPrimes = make([]pkcs1AddtionalRSAPrime, len(key.Precomputed.CRTValues)) + priv.AdditionalPrimes = make([]pkcs1AdditionalRSAPrime, len(key.Precomputed.CRTValues)) for i, values := range key.Precomputed.CRTValues { - priv.AdditionalPrimes[i].Prime = rawValueForBig(key.Primes[2+i]) - priv.AdditionalPrimes[i].Exp = rawValueForBig(values.Exp) - priv.AdditionalPrimes[i].Coeff = rawValueForBig(values.Coeff) + priv.AdditionalPrimes[i].Prime = key.Primes[2+i] + priv.AdditionalPrimes[i].Exp = values.Exp + priv.AdditionalPrimes[i].Coeff = values.Coeff } b, _ := asn1.Marshal(priv) @@ -150,35 +130,30 @@ func MarshalPKCS1PrivateKey(key *rsa.PrivateKey) []byte { type certificate struct { Raw asn1.RawContent TBSCertificate tbsCertificate - SignatureAlgorithm algorithmIdentifier + SignatureAlgorithm pkix.AlgorithmIdentifier SignatureValue asn1.BitString } type tbsCertificate struct { Raw asn1.RawContent Version int "optional,explicit,default:1,tag:0" - SerialNumber asn1.RawValue - SignatureAlgorithm algorithmIdentifier - Issuer rdnSequence + SerialNumber *big.Int + SignatureAlgorithm pkix.AlgorithmIdentifier + Issuer pkix.RDNSequence Validity validity - Subject rdnSequence + Subject pkix.RDNSequence PublicKey publicKeyInfo - UniqueId asn1.BitString "optional,tag:1" - SubjectUniqueId asn1.BitString "optional,tag:2" - Extensions []extension "optional,explicit,tag:3" + UniqueId asn1.BitString "optional,tag:1" + SubjectUniqueId asn1.BitString "optional,tag:2" + Extensions []pkix.Extension "optional,explicit,tag:3" } -type algorithmIdentifier struct { - Algorithm asn1.ObjectIdentifier +type dsaAlgorithmParameters struct { + P, Q, G *big.Int } -type rdnSequence []relativeDistinguishedNameSET - -type relativeDistinguishedNameSET []attributeTypeAndValue - -type attributeTypeAndValue struct { - Type asn1.ObjectIdentifier - Value interface{} +type dsaSignature struct { + R, S *big.Int } type validity struct { @@ -186,16 +161,11 @@ type validity struct { } type publicKeyInfo struct { - Algorithm algorithmIdentifier + Raw asn1.RawContent + Algorithm pkix.AlgorithmIdentifier PublicKey asn1.BitString } -type extension struct { - Id asn1.ObjectIdentifier - Critical bool "optional" - Value []byte -} - // RFC 5280, 4.2.1.1 type authKeyId struct { Id []byte "optional,tag:0" @@ -211,6 +181,8 @@ const ( SHA256WithRSA SHA384WithRSA SHA512WithRSA + DSAWithSHA1 + DSAWithSHA256 ) type PublicKeyAlgorithm int @@ -218,133 +190,96 @@ type PublicKeyAlgorithm int const ( UnknownPublicKeyAlgorithm PublicKeyAlgorithm = iota RSA + DSA ) -// Name represents an X.509 distinguished name. This only includes the common -// elements of a DN. Additional elements in the name are ignored. -type Name struct { - Country, Organization, OrganizationalUnit []string - Locality, Province []string - StreetAddress, PostalCode []string - SerialNumber, CommonName string -} - -func (n *Name) fillFromRDNSequence(rdns *rdnSequence) { - for _, rdn := range *rdns { - if len(rdn) == 0 { - continue - } - atv := rdn[0] - value, ok := atv.Value.(string) - if !ok { - continue - } - - t := atv.Type - if len(t) == 4 && t[0] == 2 && t[1] == 5 && t[2] == 4 { - switch t[3] { - case 3: - n.CommonName = value - case 5: - n.SerialNumber = value - case 6: - n.Country = append(n.Country, value) - case 7: - n.Locality = append(n.Locality, value) - case 8: - n.Province = append(n.Province, value) - case 9: - n.StreetAddress = append(n.StreetAddress, value) - case 10: - n.Organization = append(n.Organization, value) - case 11: - n.OrganizationalUnit = append(n.OrganizationalUnit, value) - case 17: - n.PostalCode = append(n.PostalCode, value) - } - } - } -} - +// OIDs for signature algorithms +// +// pkcs-1 OBJECT IDENTIFIER ::= { +// iso(1) member-body(2) us(840) rsadsi(113549) pkcs(1) 1 } +// +// +// RFC 3279 2.2.1 RSA Signature Algorithms +// +// md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } +// +// md5WithRSAEncryption OBJECT IDENTIFER ::= { pkcs-1 4 } +// +// sha-1WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 5 } +// +// dsaWithSha1 OBJECT IDENTIFIER ::= { +// iso(1) member-body(2) us(840) x9-57(10040) x9cm(4) 3 } +// +// +// RFC 4055 5 PKCS #1 Version 1.5 +// +// sha256WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 11 } +// +// sha384WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 12 } +// +// sha512WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 13 } +// +// +// RFC 5758 3.1 DSA Signature Algorithms +// +// dsaWithSha356 OBJECT IDENTIFER ::= { +// joint-iso-ccitt(2) country(16) us(840) organization(1) gov(101) +// algorithms(4) id-dsa-with-sha2(3) 2} +// var ( - oidCountry = []int{2, 5, 4, 6} - oidOrganization = []int{2, 5, 4, 10} - oidOrganizationalUnit = []int{2, 5, 4, 11} - oidCommonName = []int{2, 5, 4, 3} - oidSerialNumber = []int{2, 5, 4, 5} - oidLocatity = []int{2, 5, 4, 7} - oidProvince = []int{2, 5, 4, 8} - oidStreetAddress = []int{2, 5, 4, 9} - oidPostalCode = []int{2, 5, 4, 17} + oidSignatureMD2WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 2} + oidSignatureMD5WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 4} + oidSignatureSHA1WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 5} + oidSignatureSHA256WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 11} + oidSignatureSHA384WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 12} + oidSignatureSHA512WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 13} + oidSignatureDSAWithSHA1 = asn1.ObjectIdentifier{1, 2, 840, 10040, 4, 3} + oidSignatureDSAWithSHA256 = asn1.ObjectIdentifier{2, 16, 840, 1, 101, 4, 3, 2} ) -// appendRDNs appends a relativeDistinguishedNameSET to the given rdnSequence -// and returns the new value. The relativeDistinguishedNameSET contains an -// attributeTypeAndValue for each of the given values. See RFC 5280, A.1, and -// search for AttributeTypeAndValue. -func appendRDNs(in rdnSequence, values []string, oid asn1.ObjectIdentifier) rdnSequence { - if len(values) == 0 { - return in - } - - s := make([]attributeTypeAndValue, len(values)) - for i, value := range values { - s[i].Type = oid - s[i].Value = value - } - - return append(in, s) -} - -func (n Name) toRDNSequence() (ret rdnSequence) { - ret = appendRDNs(ret, n.Country, oidCountry) - ret = appendRDNs(ret, n.Organization, oidOrganization) - ret = appendRDNs(ret, n.OrganizationalUnit, oidOrganizationalUnit) - ret = appendRDNs(ret, n.Locality, oidLocatity) - ret = appendRDNs(ret, n.Province, oidProvince) - ret = appendRDNs(ret, n.StreetAddress, oidStreetAddress) - ret = appendRDNs(ret, n.PostalCode, oidPostalCode) - if len(n.CommonName) > 0 { - ret = appendRDNs(ret, []string{n.CommonName}, oidCommonName) - } - if len(n.SerialNumber) > 0 { - ret = appendRDNs(ret, []string{n.SerialNumber}, oidSerialNumber) - } - - return ret -} - -func getSignatureAlgorithmFromOID(oid []int) SignatureAlgorithm { - if len(oid) == 7 && oid[0] == 1 && oid[1] == 2 && oid[2] == 840 && - oid[3] == 113549 && oid[4] == 1 && oid[5] == 1 { - switch oid[6] { - case 2: - return MD2WithRSA - case 4: - return MD5WithRSA - case 5: - return SHA1WithRSA - case 11: - return SHA256WithRSA - case 12: - return SHA384WithRSA - case 13: - return SHA512WithRSA - } +func getSignatureAlgorithmFromOID(oid asn1.ObjectIdentifier) SignatureAlgorithm { + switch { + case oid.Equal(oidSignatureMD2WithRSA): + return MD2WithRSA + case oid.Equal(oidSignatureMD5WithRSA): + return MD5WithRSA + case oid.Equal(oidSignatureSHA1WithRSA): + return SHA1WithRSA + case oid.Equal(oidSignatureSHA256WithRSA): + return SHA256WithRSA + case oid.Equal(oidSignatureSHA384WithRSA): + return SHA384WithRSA + case oid.Equal(oidSignatureSHA512WithRSA): + return SHA512WithRSA + case oid.Equal(oidSignatureDSAWithSHA1): + return DSAWithSHA1 + case oid.Equal(oidSignatureDSAWithSHA256): + return DSAWithSHA256 } - return UnknownSignatureAlgorithm } -func getPublicKeyAlgorithmFromOID(oid []int) PublicKeyAlgorithm { - if len(oid) == 7 && oid[0] == 1 && oid[1] == 2 && oid[2] == 840 && - oid[3] == 113549 && oid[4] == 1 && oid[5] == 1 { - switch oid[6] { - case 1: - return RSA - } - } +// RFC 3279, 2.3 Public Key Algorithms +// +// pkcs-1 OBJECT IDENTIFIER ::== { iso(1) member-body(2) us(840) +// rsadsi(113549) pkcs(1) 1 } +// +// rsaEncryption OBJECT IDENTIFIER ::== { pkcs1-1 1 } +// +// id-dsa OBJECT IDENTIFIER ::== { iso(1) member-body(2) us(840) +// x9-57(10040) x9cm(4) 1 } +var ( + oidPublicKeyRsa = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 1} + oidPublicKeyDsa = asn1.ObjectIdentifier{1, 2, 840, 10040, 4, 1} +) +func getPublicKeyAlgorithmFromOID(oid asn1.ObjectIdentifier) PublicKeyAlgorithm { + switch { + case oid.Equal(oidPublicKeyRsa): + return RSA + case oid.Equal(oidPublicKeyDsa): + return DSA + } return UnknownPublicKeyAlgorithm } @@ -402,8 +337,10 @@ const ( // A Certificate represents an X.509 certificate. type Certificate struct { - Raw []byte // Complete ASN.1 DER content (certificate, signature algorithm and signature). - RawTBSCertificate []byte // Certificate part of raw ASN.1 DER content. + Raw []byte // Complete ASN.1 DER content (certificate, signature algorithm and signature). + RawTBSCertificate []byte // Certificate part of raw ASN.1 DER content. + RawSubjectPublicKeyInfo []byte // DER encoded SubjectPublicKeyInfo. + Signature []byte SignatureAlgorithm SignatureAlgorithm @@ -411,9 +348,9 @@ type Certificate struct { PublicKey interface{} Version int - SerialNumber []byte - Issuer Name - Subject Name + SerialNumber *big.Int + Issuer pkix.Name + Subject pkix.Name NotBefore, NotAfter *time.Time // Validity bounds. KeyUsage KeyUsage @@ -482,26 +419,58 @@ func (c *Certificate) CheckSignatureFrom(parent *Certificate) (err os.Error) { // TODO(agl): don't ignore the path length constraint. - var h hash.Hash + return parent.CheckSignature(c.SignatureAlgorithm, c.RawTBSCertificate, c.Signature) +} + +// CheckSignature verifies that signature is a valid signature over signed from +// c's public key. +func (c *Certificate) CheckSignature(algo SignatureAlgorithm, signed, signature []byte) (err os.Error) { var hashType crypto.Hash - switch c.SignatureAlgorithm { - case SHA1WithRSA: - h = sha1.New() + switch algo { + case SHA1WithRSA, DSAWithSHA1: hashType = crypto.SHA1 + case SHA256WithRSA, DSAWithSHA256: + hashType = crypto.SHA256 + case SHA384WithRSA: + hashType = crypto.SHA384 + case SHA512WithRSA: + hashType = crypto.SHA512 default: return UnsupportedAlgorithmError{} } - pub, ok := parent.PublicKey.(*rsa.PublicKey) - if !ok { + h := hashType.New() + if h == nil { return UnsupportedAlgorithmError{} } - h.Write(c.RawTBSCertificate) + h.Write(signed) digest := h.Sum() - return rsa.VerifyPKCS1v15(pub, hashType, digest, c.Signature) + switch pub := c.PublicKey.(type) { + case *rsa.PublicKey: + return rsa.VerifyPKCS1v15(pub, hashType, digest, signature) + case *dsa.PublicKey: + dsaSig := new(dsaSignature) + if _, err := asn1.Unmarshal(signature, dsaSig); err != nil { + return err + } + if dsaSig.R.Sign() <= 0 || dsaSig.S.Sign() <= 0 { + return os.ErrorString("DSA signature contained zero or negative values") + } + if !dsa.Verify(pub, digest, dsaSig.R, dsaSig.S) { + return os.ErrorString("DSA verification failure") + } + return + } + return UnsupportedAlgorithmError{} +} + +// CheckCRLSignature checks that the signature in crl is from c. +func (c *Certificate) CheckCRLSignature(crl *pkix.CertificateList) (err os.Error) { + algo := getSignatureAlgorithmFromOID(crl.SignatureAlgorithm.Algorithm) + return c.CheckSignature(algo, crl.TBSCertList.Raw, crl.SignatureValue.RightAlign()) } type UnhandledCriticalExtension struct{} @@ -516,7 +485,7 @@ type basicConstraints struct { } type rsaPublicKey struct { - N asn1.RawValue + N *big.Int E int } @@ -538,7 +507,8 @@ type generalSubtree struct { Max int "optional,tag:1" } -func parsePublicKey(algo PublicKeyAlgorithm, asn1Data []byte) (interface{}, os.Error) { +func parsePublicKey(algo PublicKeyAlgorithm, keyData *publicKeyInfo) (interface{}, os.Error) { + asn1Data := keyData.PublicKey.RightAlign() switch algo { case RSA: p := new(rsaPublicKey) @@ -547,19 +517,38 @@ func parsePublicKey(algo PublicKeyAlgorithm, asn1Data []byte) (interface{}, os.E return nil, err } - if !rawValueIsInteger(&p.N) { - return nil, asn1.StructuralError{"tags don't match"} - } - pub := &rsa.PublicKey{ E: p.E, - N: new(big.Int).SetBytes(p.N.Bytes), + N: p.N, + } + return pub, nil + case DSA: + var p *big.Int + _, err := asn1.Unmarshal(asn1Data, &p) + if err != nil { + return nil, err + } + paramsData := keyData.Algorithm.Parameters.FullBytes + params := new(dsaAlgorithmParameters) + _, err = asn1.Unmarshal(paramsData, params) + if err != nil { + return nil, err + } + if p.Sign() <= 0 || params.P.Sign() <= 0 || params.Q.Sign() <= 0 || params.G.Sign() <= 0 { + return nil, os.ErrorString("zero or negative DSA parameter") + } + pub := &dsa.PublicKey{ + Parameters: dsa.Parameters{ + P: params.P, + Q: params.Q, + G: params.G, + }, + Y: p, } return pub, nil default: return nil, nil } - panic("unreachable") } @@ -567,6 +556,7 @@ func parseCertificate(in *certificate) (*Certificate, os.Error) { out := new(Certificate) out.Raw = in.Raw out.RawTBSCertificate = in.TBSCertificate.Raw + out.RawSubjectPublicKeyInfo = in.TBSCertificate.PublicKey.Raw out.Signature = in.SignatureValue.RightAlign() out.SignatureAlgorithm = @@ -575,15 +565,19 @@ func parseCertificate(in *certificate) (*Certificate, os.Error) { out.PublicKeyAlgorithm = getPublicKeyAlgorithmFromOID(in.TBSCertificate.PublicKey.Algorithm.Algorithm) var err os.Error - out.PublicKey, err = parsePublicKey(out.PublicKeyAlgorithm, in.TBSCertificate.PublicKey.PublicKey.RightAlign()) + out.PublicKey, err = parsePublicKey(out.PublicKeyAlgorithm, &in.TBSCertificate.PublicKey) if err != nil { return nil, err } + if in.TBSCertificate.SerialNumber.Sign() < 0 { + return nil, os.ErrorString("negative serial number") + } + out.Version = in.TBSCertificate.Version + 1 - out.SerialNumber = in.TBSCertificate.SerialNumber.Bytes - out.Issuer.fillFromRDNSequence(&in.TBSCertificate.Issuer) - out.Subject.fillFromRDNSequence(&in.TBSCertificate.Subject) + out.SerialNumber = in.TBSCertificate.SerialNumber + out.Issuer.FillFromRDNSequence(&in.TBSCertificate.Issuer) + out.Subject.FillFromRDNSequence(&in.TBSCertificate.Subject) out.NotBefore = in.TBSCertificate.Validity.NotBefore out.NotAfter = in.TBSCertificate.Validity.NotAfter @@ -607,13 +601,13 @@ func parseCertificate(in *certificate) (*Certificate, os.Error) { } case 19: // RFC 5280, 4.2.1.9 - var constriants basicConstraints - _, err := asn1.Unmarshal(e.Value, &constriants) + var constraints basicConstraints + _, err := asn1.Unmarshal(e.Value, &constraints) if err == nil { out.BasicConstraintsValid = true - out.IsCA = constriants.IsCA - out.MaxPathLen = constriants.MaxPathLen + out.IsCA = constraints.IsCA + out.MaxPathLen = constraints.MaxPathLen continue } case 17: @@ -841,8 +835,8 @@ var ( oidExtensionNameConstraints = []int{2, 5, 29, 30} ) -func buildExtensions(template *Certificate) (ret []extension, err os.Error) { - ret = make([]extension, 7 /* maximum number of elements. */ ) +func buildExtensions(template *Certificate) (ret []pkix.Extension, err os.Error) { + ret = make([]pkix.Extension, 7 /* maximum number of elements. */ ) n := 0 if template.KeyUsage != 0 { @@ -959,7 +953,7 @@ var ( // The returned slice is the certificate in DER encoding. func CreateCertificate(rand io.Reader, template, parent *Certificate, pub *rsa.PublicKey, priv *rsa.PrivateKey) (cert []byte, err os.Error) { asn1PublicKey, err := asn1.Marshal(rsaPublicKey{ - N: asn1.RawValue{Tag: 2, Bytes: pub.N.Bytes()}, + N: pub.N, E: pub.E, }) if err != nil { @@ -978,12 +972,12 @@ func CreateCertificate(rand io.Reader, template, parent *Certificate, pub *rsa.P encodedPublicKey := asn1.BitString{BitLength: len(asn1PublicKey) * 8, Bytes: asn1PublicKey} c := tbsCertificate{ Version: 2, - SerialNumber: asn1.RawValue{Bytes: template.SerialNumber, Tag: 2}, - SignatureAlgorithm: algorithmIdentifier{oidSHA1WithRSA}, - Issuer: parent.Subject.toRDNSequence(), + SerialNumber: template.SerialNumber, + SignatureAlgorithm: pkix.AlgorithmIdentifier{Algorithm: oidSHA1WithRSA}, + Issuer: parent.Subject.ToRDNSequence(), Validity: validity{template.NotBefore, template.NotAfter}, - Subject: template.Subject.toRDNSequence(), - PublicKey: publicKeyInfo{algorithmIdentifier{oidRSA}, encodedPublicKey}, + Subject: template.Subject.ToRDNSequence(), + PublicKey: publicKeyInfo{nil, pkix.AlgorithmIdentifier{Algorithm: oidRSA}, encodedPublicKey}, Extensions: extensions, } @@ -1006,8 +1000,75 @@ func CreateCertificate(rand io.Reader, template, parent *Certificate, pub *rsa.P cert, err = asn1.Marshal(certificate{ nil, c, - algorithmIdentifier{oidSHA1WithRSA}, + pkix.AlgorithmIdentifier{Algorithm: oidSHA1WithRSA}, asn1.BitString{Bytes: signature, BitLength: len(signature) * 8}, }) return } + +// pemCRLPrefix is the magic string that indicates that we have a PEM encoded +// CRL. +var pemCRLPrefix = []byte("-----BEGIN X509 CRL") +// pemType is the type of a PEM encoded CRL. +var pemType = "X509 CRL" + +// ParseCRL parses a CRL from the given bytes. It's often the case that PEM +// encoded CRLs will appear where they should be DER encoded, so this function +// will transparently handle PEM encoding as long as there isn't any leading +// garbage. +func ParseCRL(crlBytes []byte) (certList *pkix.CertificateList, err os.Error) { + if bytes.HasPrefix(crlBytes, pemCRLPrefix) { + block, _ := pem.Decode(crlBytes) + if block != nil && block.Type == pemType { + crlBytes = block.Bytes + } + } + return ParseDERCRL(crlBytes) +} + +// ParseDERCRL parses a DER encoded CRL from the given bytes. +func ParseDERCRL(derBytes []byte) (certList *pkix.CertificateList, err os.Error) { + certList = new(pkix.CertificateList) + _, err = asn1.Unmarshal(derBytes, certList) + if err != nil { + certList = nil + } + return +} + +// CreateCRL returns a DER encoded CRL, signed by this Certificate, that +// contains the given list of revoked certificates. +func (c *Certificate) CreateCRL(rand io.Reader, priv *rsa.PrivateKey, revokedCerts []pkix.RevokedCertificate, now, expiry *time.Time) (crlBytes []byte, err os.Error) { + tbsCertList := pkix.TBSCertificateList{ + Version: 2, + Signature: pkix.AlgorithmIdentifier{ + Algorithm: oidSignatureSHA1WithRSA, + }, + Issuer: c.Subject.ToRDNSequence(), + ThisUpdate: now, + NextUpdate: expiry, + RevokedCertificates: revokedCerts, + } + + tbsCertListContents, err := asn1.Marshal(tbsCertList) + if err != nil { + return + } + + h := sha1.New() + h.Write(tbsCertListContents) + digest := h.Sum() + + signature, err := rsa.SignPKCS1v15(rand, priv, crypto.SHA1, digest) + if err != nil { + return + } + + return asn1.Marshal(pkix.CertificateList{ + TBSCertList: tbsCertList, + SignatureAlgorithm: pkix.AlgorithmIdentifier{ + Algorithm: oidSignatureSHA1WithRSA, + }, + SignatureValue: asn1.BitString{Bytes: signature, BitLength: len(signature) * 8}, + }) +} diff --git a/src/pkg/crypto/x509/x509_test.go b/src/pkg/crypto/x509/x509_test.go index a42113add..dc216505e 100644 --- a/src/pkg/crypto/x509/x509_test.go +++ b/src/pkg/crypto/x509/x509_test.go @@ -7,8 +7,11 @@ package x509 import ( "asn1" "big" + "crypto/dsa" "crypto/rand" "crypto/rsa" + "crypto/x509/pkix" + "encoding/base64" "encoding/hex" "encoding/pem" "testing" @@ -54,6 +57,12 @@ func fromBase10(base10 string) *big.Int { return i } +func bigFromHexString(s string) *big.Int { + ret := new(big.Int) + ret.SetString(s, 16) + return ret +} + var rsaPrivateKey = &rsa.PrivateKey{ PublicKey: rsa.PublicKey{ N: bigFromString("9353930466774385905609975137998169297361893554149986716853295022578535724979677252958524466350471210367835187480748268864277464700638583474144061408845077"), @@ -200,8 +209,8 @@ func TestCreateSelfSignedCertificate(t *testing.T) { } template := Certificate{ - SerialNumber: []byte{1}, - Subject: Name{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ CommonName: "test.example.com", Organization: []string{"Acme Co"}, }, @@ -245,3 +254,178 @@ func TestCreateSelfSignedCertificate(t *testing.T) { return } } + +// Self-signed certificate using DSA with SHA1 +var dsaCertPem = `-----BEGIN CERTIFICATE----- +MIIEDTCCA82gAwIBAgIJALHPghaoxeDhMAkGByqGSM44BAMweTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAk5DMQ8wDQYDVQQHEwZOZXd0b24xFDASBgNVBAoTC0dvb2ds +ZSwgSW5jMRIwEAYDVQQDEwlKb24gQWxsaWUxIjAgBgkqhkiG9w0BCQEWE2pvbmFs +bGllQGdvb2dsZS5jb20wHhcNMTEwNTE0MDMwMTQ1WhcNMTEwNjEzMDMwMTQ1WjB5 +MQswCQYDVQQGEwJVUzELMAkGA1UECBMCTkMxDzANBgNVBAcTBk5ld3RvbjEUMBIG +A1UEChMLR29vZ2xlLCBJbmMxEjAQBgNVBAMTCUpvbiBBbGxpZTEiMCAGCSqGSIb3 +DQEJARYTam9uYWxsaWVAZ29vZ2xlLmNvbTCCAbcwggEsBgcqhkjOOAQBMIIBHwKB +gQC8hLUnQ7FpFYu4WXTj6DKvXvz8QrJkNJCVMTpKAT7uBpobk32S5RrPKXocd4gN +8lyGB9ggS03EVlEwXvSmO0DH2MQtke2jl9j1HLydClMf4sbx5V6TV9IFw505U1iW +jL7awRMgxge+FsudtJK254FjMFo03ZnOQ8ZJJ9E6AEDrlwIVAJpnBn9moyP11Ox5 +Asc/5dnjb6dPAoGBAJFHd4KVv1iTVCvEG6gGiYop5DJh28hUQcN9kul+2A0yPUSC +X93oN00P8Vh3eYgSaCWZsha7zDG53MrVJ0Zf6v/X/CoZNhLldeNOepivTRAzn+Rz +kKUYy5l1sxYLHQKF0UGNCXfFKZT0PCmgU+PWhYNBBMn6/cIh44vp85ideo5CA4GE +AAKBgFmifCafzeRaohYKXJgMGSEaggCVCRq5xdyDCat+wbOkjC4mfG01/um3G8u5 +LxasjlWRKTR/tcAL7t0QuokVyQaYdVypZXNaMtx1db7YBuHjj3aP+8JOQRI9xz8c +bp5NDJ5pISiFOv4p3GZfqZPcqckDt78AtkQrmnal2txhhjF6o4HeMIHbMB0GA1Ud +DgQWBBQVyyr7hO11ZFFpWX50298Sa3V+rzCBqwYDVR0jBIGjMIGggBQVyyr7hO11 +ZFFpWX50298Sa3V+r6F9pHsweTELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAk5DMQ8w +DQYDVQQHEwZOZXd0b24xFDASBgNVBAoTC0dvb2dsZSwgSW5jMRIwEAYDVQQDEwlK +b24gQWxsaWUxIjAgBgkqhkiG9w0BCQEWE2pvbmFsbGllQGdvb2dsZS5jb22CCQCx +z4IWqMXg4TAMBgNVHRMEBTADAQH/MAkGByqGSM44BAMDLwAwLAIUPtn/5j8Q1jJI +7ggOIsgrhgUdjGQCFCsmDq1H11q9+9Wp9IMeGrTSKHIM +-----END CERTIFICATE----- +` + +func TestParseCertificateWithDsaPublicKey(t *testing.T) { + expectedKey := &dsa.PublicKey{ + Parameters: dsa.Parameters{ + P: bigFromHexString("00BC84B52743B169158BB85974E3E832AF5EFCFC42B264349095313A4A013EEE069A1B937D92E51ACF297A1C77880DF25C8607D8204B4DC45651305EF4A63B40C7D8C42D91EDA397D8F51CBC9D0A531FE2C6F1E55E9357D205C39D395358968CBEDAC11320C607BE16CB9DB492B6E78163305A34DD99CE43C64927D13A0040EB97"), + Q: bigFromHexString("009A67067F66A323F5D4EC7902C73FE5D9E36FA74F"), + G: bigFromHexString("009147778295BF5893542BC41BA806898A29E43261DBC85441C37D92E97ED80D323D44825FDDE8374D0FF15877798812682599B216BBCC31B9DCCAD527465FEAFFD7FC2A193612E575E34E7A98AF4D10339FE47390A518CB9975B3160B1D0285D1418D0977C52994F43C29A053E3D685834104C9FAFDC221E38BE9F3989D7A8E42"), + }, + Y: bigFromHexString("59A27C269FCDE45AA2160A5C980C19211A820095091AB9C5DC8309AB7EC1B3A48C2E267C6D35FEE9B71BCBB92F16AC8E559129347FB5C00BEEDD10BA8915C90698755CA965735A32DC7575BED806E1E38F768FFBC24E41123DC73F1C6E9E4D0C9E692128853AFE29DC665FA993DCA9C903B7BF00B6442B9A76A5DADC6186317A"), + } + pemBlock, _ := pem.Decode([]byte(dsaCertPem)) + cert, err := ParseCertificate(pemBlock.Bytes) + if err != nil { + t.Fatalf("Failed to parse certificate: %s", err) + } + if cert.PublicKeyAlgorithm != DSA { + t.Errorf("Parsed key algorithm was not DSA") + } + parsedKey, ok := cert.PublicKey.(*dsa.PublicKey) + if !ok { + t.Fatalf("Parsed key was not a DSA key: %s", err) + } + if expectedKey.Y.Cmp(parsedKey.Y) != 0 || + expectedKey.P.Cmp(parsedKey.P) != 0 || + expectedKey.Q.Cmp(parsedKey.Q) != 0 || + expectedKey.G.Cmp(parsedKey.G) != 0 { + t.Fatal("Parsed key differs from expected key") + } +} + +func TestParseCertificateWithDSASignatureAlgorithm(t *testing.T) { + pemBlock, _ := pem.Decode([]byte(dsaCertPem)) + cert, err := ParseCertificate(pemBlock.Bytes) + if err != nil { + t.Fatalf("Failed to parse certificate: %s", err) + } + if cert.SignatureAlgorithm != DSAWithSHA1 { + t.Errorf("Parsed signature algorithm was not DSAWithSHA1") + } +} + +func TestVerifyCertificateWithDSASignature(t *testing.T) { + pemBlock, _ := pem.Decode([]byte(dsaCertPem)) + cert, err := ParseCertificate(pemBlock.Bytes) + if err != nil { + t.Fatalf("Failed to parse certificate: %s", err) + } + // test cert is self-signed + if err = cert.CheckSignatureFrom(cert); err != nil { + t.Fatalf("DSA Certificate verfication failed: %s", err) + } +} + +const pemCertificate = `-----BEGIN CERTIFICATE----- +MIIB5DCCAZCgAwIBAgIBATALBgkqhkiG9w0BAQUwLTEQMA4GA1UEChMHQWNtZSBDbzEZMBcGA1UE +AxMQdGVzdC5leGFtcGxlLmNvbTAeFw03MDAxMDEwMDE2NDBaFw03MDAxMDIwMzQ2NDBaMC0xEDAO +BgNVBAoTB0FjbWUgQ28xGTAXBgNVBAMTEHRlc3QuZXhhbXBsZS5jb20wWjALBgkqhkiG9w0BAQED +SwAwSAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0fd7Ai2KW5ToIwzFo +fvJcS/STa6HA5gQenRUCAwEAAaOBnjCBmzAOBgNVHQ8BAf8EBAMCAAQwDwYDVR0TAQH/BAUwAwEB +/zANBgNVHQ4EBgQEAQIDBDAPBgNVHSMECDAGgAQBAgMEMBsGA1UdEQQUMBKCEHRlc3QuZXhhbXBs +ZS5jb20wDwYDVR0gBAgwBjAEBgIqAzAqBgNVHR4EIzAhoB8wDoIMLmV4YW1wbGUuY29tMA2CC2V4 +YW1wbGUuY29tMAsGCSqGSIb3DQEBBQNBAHKZKoS1wEQOGhgklx4+/yFYQlnqwKXvar/ZecQvJwui +0seMQnwBhwdBkHfVIU2Fu5VUMRyxlf0ZNaDXcpU581k= +-----END CERTIFICATE-----` + +func TestCRLCreation(t *testing.T) { + block, _ := pem.Decode([]byte(pemPrivateKey)) + priv, _ := ParsePKCS1PrivateKey(block.Bytes) + block, _ = pem.Decode([]byte(pemCertificate)) + cert, _ := ParseCertificate(block.Bytes) + + now := time.SecondsToUTC(1000) + expiry := time.SecondsToUTC(10000) + + revokedCerts := []pkix.RevokedCertificate{ + { + SerialNumber: big.NewInt(1), + RevocationTime: now, + }, + { + SerialNumber: big.NewInt(42), + RevocationTime: now, + }, + } + + crlBytes, err := cert.CreateCRL(rand.Reader, priv, revokedCerts, now, expiry) + if err != nil { + t.Errorf("error creating CRL: %s", err) + } + + _, err = ParseDERCRL(crlBytes) + if err != nil { + t.Errorf("error reparsing CRL: %s", err) + } +} + +func fromBase64(in string) []byte { + out := make([]byte, base64.StdEncoding.DecodedLen(len(in))) + _, err := base64.StdEncoding.Decode(out, []byte(in)) + if err != nil { + panic("failed to base64 decode") + } + return out +} + +func TestParseDERCRL(t *testing.T) { + derBytes := fromBase64(derCRLBase64) + certList, err := ParseDERCRL(derBytes) + if err != nil { + t.Errorf("error parsing: %s", err) + return + } + numCerts := len(certList.TBSCertList.RevokedCertificates) + expected := 88 + if numCerts != expected { + t.Errorf("bad number of revoked certificates. got: %d want: %d", numCerts, expected) + } + + if certList.HasExpired(1302517272) { + t.Errorf("CRL has expired (but shouldn't have)") + } + + // Can't check the signature here without a package cycle. +} + +func TestParsePEMCRL(t *testing.T) { + pemBytes := fromBase64(pemCRLBase64) + certList, err := ParseCRL(pemBytes) + if err != nil { + t.Errorf("error parsing: %s", err) + return + } + numCerts := len(certList.TBSCertList.RevokedCertificates) + expected := 2 + if numCerts != expected { + t.Errorf("bad number of revoked certificates. got: %d want: %d", numCerts, expected) + } + + if certList.HasExpired(1302517272) { + t.Errorf("CRL has expired (but shouldn't have)") + } + + // Can't check the signature here without a package cycle. +} + +const derCRLBase64 = "MIINqzCCDJMCAQEwDQYJKoZIhvcNAQEFBQAwVjEZMBcGA1UEAxMQUEtJIEZJTk1FQ0NBTklDQTEVMBMGA1UEChMMRklOTUVDQ0FOSUNBMRUwEwYDVQQLEwxGSU5NRUNDQU5JQ0ExCzAJBgNVBAYTAklUFw0xMTA1MDQxNjU3NDJaFw0xMTA1MDQyMDU3NDJaMIIMBzAhAg4Ze1od49Lt1qIXBydAzhcNMDkwNzE2MDg0MzIyWjAAMCECDl0HSL9bcZ1Ci/UHJ0DPFw0wOTA3MTYwODQzMTNaMAAwIQIOESB9tVAmX3cY7QcnQNAXDTA5MDcxNjA4NDUyMlowADAhAg4S1tGAQ3mHt8uVBydA1RcNMDkwODA0MTUyNTIyWjAAMCECDlQ249Y7vtC25ScHJ0DWFw0wOTA4MDQxNTI1MzdaMAAwIQIOISMop3NkA4PfYwcnQNkXDTA5MDgwNDExMDAzNFowADAhAg56/BMoS29KEShTBydA2hcNMDkwODA0MTEwMTAzWjAAMCECDnBp/22HPH5CSWoHJ0DbFw0wOTA4MDQxMDU0NDlaMAAwIQIOV9IP+8CD8bK+XAcnQNwXDTA5MDgwNDEwNTcxN1owADAhAg4v5aRz0IxWqYiXBydA3RcNMDkwODA0MTA1NzQ1WjAAMCECDlOU34VzvZAybQwHJ0DeFw0wOTA4MDQxMDU4MjFaMAAwIAINO4CD9lluIxcwBydBAxcNMDkwNzIyMTUzMTU5WjAAMCECDgOllfO8Y1QA7/wHJ0ExFw0wOTA3MjQxMTQxNDNaMAAwIQIOJBX7jbiCdRdyjgcnQUQXDTA5MDkxNjA5MzAwOFowADAhAg5iYSAgmDrlH/RZBydBRRcNMDkwOTE2MDkzMDE3WjAAMCECDmu6k6srP3jcMaQHJ0FRFw0wOTA4MDQxMDU2NDBaMAAwIQIOX8aHlO0V+WVH4QcnQVMXDTA5MDgwNDEwNTcyOVowADAhAg5flK2rg3NnsRgDBydBzhcNMTEwMjAxMTUzMzQ2WjAAMCECDg35yJDL1jOPTgoHJ0HPFw0xMTAyMDExNTM0MjZaMAAwIQIOMyFJ6+e9iiGVBQcnQdAXDTA5MDkxODEzMjAwNVowADAhAg5Emb/Oykucmn8fBydB1xcNMDkwOTIxMTAxMDQ3WjAAMCECDjQKCncV+MnUavMHJ0HaFw0wOTA5MjIwODE1MjZaMAAwIQIOaxiFUt3dpd+tPwcnQfQXDTEwMDYxODA4NDI1MVowADAhAg5G7P8nO0tkrMt7BydB9RcNMTAwNjE4MDg0MjMwWjAAMCECDmTCC3SXhmDRst4HJ0H2Fw0wOTA5MjgxMjA3MjBaMAAwIQIOHoGhUr/pRwzTKgcnQfcXDTA5MDkyODEyMDcyNFowADAhAg50wrcrCiw8mQmPBydCBBcNMTAwMjE2MTMwMTA2WjAAMCECDifWmkvwyhEqwEcHJ0IFFw0xMDAyMTYxMzAxMjBaMAAwIQIOfgPmlW9fg+osNgcnQhwXDTEwMDQxMzA5NTIwMFowADAhAg4YHAGuA6LgCk7tBydCHRcNMTAwNDEzMDk1MTM4WjAAMCECDi1zH1bxkNJhokAHJ0IsFw0xMDA0MTMwOTU5MzBaMAAwIQIOMipNccsb/wo2fwcnQi0XDTEwMDQxMzA5NTkwMFowADAhAg46lCmvPl4GpP6ABydCShcNMTAwMTE5MDk1MjE3WjAAMCECDjaTcaj+wBpcGAsHJ0JLFw0xMDAxMTkwOTUyMzRaMAAwIQIOOMC13EOrBuxIOQcnQloXDTEwMDIwMTA5NDcwNVowADAhAg5KmZl+krz4RsmrBydCWxcNMTAwMjAxMDk0NjQwWjAAMCECDmLG3zQJ/fzdSsUHJ0JiFw0xMDAzMDEwOTUxNDBaMAAwIQIOP39ksgHdojf4owcnQmMXDTEwMDMwMTA5NTExN1owADAhAg4LDQzvWNRlD6v9BydCZBcNMTAwMzAxMDk0NjIyWjAAMCECDkmNfeclaFhIaaUHJ0JlFw0xMDAzMDEwOTQ2MDVaMAAwIQIOT/qWWfpH/m8NTwcnQpQXDTEwMDUxMTA5MTgyMVowADAhAg5m/ksYxvCEgJSvBydClRcNMTAwNTExMDkxODAxWjAAMCECDgvf3Ohq6JOPU9AHJ0KWFw0xMDA1MTEwOTIxMjNaMAAwIQIOKSPas10z4jNVIQcnQpcXDTEwMDUxMTA5MjEwMlowADAhAg4mCWmhoZ3lyKCDBydCohcNMTEwNDI4MTEwMjI1WjAAMCECDkeiyRsBMK0Gvr4HJ0KjFw0xMTA0MjgxMTAyMDdaMAAwIQIOa09b/nH2+55SSwcnQq4XDTExMDQwMTA4Mjk0NlowADAhAg5O7M7iq7gGplr1BydCrxcNMTEwNDAxMDgzMDE3WjAAMCECDjlT6mJxUjTvyogHJ0K1Fw0xMTAxMjcxNTQ4NTJaMAAwIQIODS/l4UUFLe21NAcnQrYXDTExMDEyNzE1NDgyOFowADAhAg5lPRA0XdOUF6lSBydDHhcNMTEwMTI4MTQzNTA1WjAAMCECDixKX4fFGGpENwgHJ0MfFw0xMTAxMjgxNDM1MzBaMAAwIQIORNBkqsPnpKTtbAcnQ08XDTEwMDkwOTA4NDg0MlowADAhAg5QL+EMM3lohedEBydDUBcNMTAwOTA5MDg0ODE5WjAAMCECDlhDnHK+HiTRAXcHJ0NUFw0xMDEwMTkxNjIxNDBaMAAwIQIOdBFqAzq/INz53gcnQ1UXDTEwMTAxOTE2MjA0NFowADAhAg4OjR7s8MgKles1BydDWhcNMTEwMTI3MTY1MzM2WjAAMCECDmfR/elHee+d0SoHJ0NbFw0xMTAxMjcxNjUzNTZaMAAwIQIOBTKv2ui+KFMI+wcnQ5YXDTEwMDkxNTEwMjE1N1owADAhAg49F3c/GSah+oRUBydDmxcNMTEwMTI3MTczMjMzWjAAMCECDggv4I61WwpKFMMHJ0OcFw0xMTAxMjcxNzMyNTVaMAAwIQIOXx/Y8sEvwS10LAcnQ6UXDTExMDEyODExMjkzN1owADAhAg5LSLbnVrSKaw/9BydDphcNMTEwMTI4MTEyOTIwWjAAMCECDmFFoCuhKUeACQQHJ0PfFw0xMTAxMTExMDE3MzdaMAAwIQIOQTDdFh2fSPF6AAcnQ+AXDTExMDExMTEwMTcxMFowADAhAg5B8AOXX61FpvbbBydD5RcNMTAxMDA2MTAxNDM2WjAAMCECDh41P2Gmi7PkwI4HJ0PmFw0xMDEwMDYxMDE2MjVaMAAwIQIOWUHGLQCd+Ale9gcnQ/0XDTExMDUwMjA3NTYxMFowADAhAg5Z2c9AYkikmgWOBydD/hcNMTEwNTAyMDc1NjM0WjAAMCECDmf/UD+/h8nf+74HJ0QVFw0xMTA0MTUwNzI4MzNaMAAwIQIOICvj4epy3MrqfwcnRBYXDTExMDQxNTA3Mjg1NlowADAhAg4bouRMfOYqgv4xBydEHxcNMTEwMzA4MTYyNDI1WjAAMCECDhebWHGoKiTp7pEHJ0QgFw0xMTAzMDgxNjI0NDhaMAAwIQIOX+qnxxAqJ8LtawcnRDcXDTExMDEzMTE1MTIyOFowADAhAg4j0fICqZ+wkOdqBydEOBcNMTEwMTMxMTUxMTQxWjAAMCECDhmXjsV4SUpWtAMHJ0RLFw0xMTAxMjgxMTI0MTJaMAAwIQIODno/w+zG43kkTwcnREwXDTExMDEyODExMjM1MlowADAhAg4b1gc88767Fr+LBydETxcNMTEwMTI4MTEwMjA4WjAAMCECDn+M3Pa1w2nyFeUHJ0RQFw0xMTAxMjgxMDU4NDVaMAAwIQIOaduoyIH61tqybAcnRJUXDTEwMTIxNTA5NDMyMlowADAhAg4nLqQPkyi3ESAKBydElhcNMTAxMjE1MDk0MzM2WjAAMCECDi504NIMH8578gQHJ0SbFw0xMTAyMTQxNDA1NDFaMAAwIQIOGuaM8PDaC5u1egcnRJwXDTExMDIxNDE0MDYwNFowADAhAg4ehYq/BXGnB5PWBydEnxcNMTEwMjA0MDgwOTUxWjAAMCECDkSD4eS4FxW5H20HJ0SgFw0xMTAyMDQwODA5MjVaMAAwIQIOOCcb6ilYObt1egcnRKEXDTExMDEyNjEwNDEyOVowADAhAg58tISWCCwFnKGnBydEohcNMTEwMjA0MDgxMzQyWjAAMCECDn5rjtabY/L/WL0HJ0TJFw0xMTAyMDQxMTAzNDFaMAAwDQYJKoZIhvcNAQEFBQADggEBAGnF2Gs0+LNiYCW1Ipm83OXQYP/bd5tFFRzyz3iepFqNfYs4D68/QihjFoRHQoXEB0OEe1tvaVnnPGnEOpi6krwekquMxo4H88B5SlyiFIqemCOIss0SxlCFs69LmfRYvPPvPEhoXtQ3ZThe0UvKG83GOklhvGl6OaiRf4Mt+m8zOT4Wox/j6aOBK6cw6qKCdmD+Yj1rrNqFGg1CnSWMoD6S6mwNgkzwdBUJZ22BwrzAAo4RHa2Uy3ef1FjwD0XtU5N3uDSxGGBEDvOe5z82rps3E22FpAA8eYl8kaXtmWqyvYU0epp4brGuTxCuBMCAsxt/OjIjeNNQbBGkwxgfYA0=" + +const pemCRLBase64 = "LS0tLS1CRUdJTiBYNTA5IENSTC0tLS0tDQpNSUlCOWpDQ0FWOENBUUV3RFFZSktvWklodmNOQVFFRkJRQXdiREVhTUJnR0ExVUVDaE1SVWxOQklGTmxZM1Z5DQphWFI1SUVsdVl5NHhIakFjQmdOVkJBTVRGVkpUUVNCUWRXSnNhV01nVW05dmRDQkRRU0IyTVRFdU1Dd0dDU3FHDQpTSWIzRFFFSkFSWWZjbk5oYTJWdmJuSnZiM1J6YVdkdVFISnpZWE5sWTNWeWFYUjVMbU52YlJjTk1URXdNakl6DQpNVGt5T0RNd1doY05NVEV3T0RJeU1Ua3lPRE13V2pDQmpEQktBaEVBckRxb2g5RkhKSFhUN09QZ3V1bjQrQmNODQpNRGt4TVRBeU1UUXlOekE1V2pBbU1Bb0dBMVVkRlFRRENnRUpNQmdHQTFVZEdBUVJHQTh5TURBNU1URXdNakUwDQpNalExTlZvd1BnSVJBTEd6blowOTVQQjVhQU9MUGc1N2ZNTVhEVEF5TVRBeU16RTBOVEF4TkZvd0dqQVlCZ05WDQpIUmdFRVJnUE1qQXdNakV3TWpNeE5EVXdNVFJhb0RBd0xqQWZCZ05WSFNNRUdEQVdnQlQxVERGNlVRTS9MTmVMDQpsNWx2cUhHUXEzZzltekFMQmdOVkhSUUVCQUlDQUlRd0RRWUpLb1pJaHZjTkFRRUZCUUFEZ1lFQUZVNUFzNk16DQpxNVBSc2lmYW9iUVBHaDFhSkx5QytNczVBZ2MwYld5QTNHQWR4dXI1U3BQWmVSV0NCamlQL01FSEJXSkNsQkhQDQpHUmNxNXlJZDNFakRrYUV5eFJhK2k2N0x6dmhJNmMyOUVlNks5cFNZd2ppLzdSVWhtbW5Qclh0VHhsTDBsckxyDQptUVFKNnhoRFJhNUczUUE0Q21VZHNITnZicnpnbUNZcHZWRT0NCi0tLS0tRU5EIFg1MDkgQ1JMLS0tLS0NCg0K" diff --git a/src/pkg/crypto/xtea/block.go b/src/pkg/crypto/xtea/block.go index 3ac36d038..bf5d24599 100644 --- a/src/pkg/crypto/xtea/block.go +++ b/src/pkg/crypto/xtea/block.go @@ -22,7 +22,7 @@ func blockToUint32(src []byte) (uint32, uint32) { return r0, r1 } -// uint32ToBlock writes two unint32s into an 8 byte data block. +// uint32ToBlock writes two uint32s into an 8 byte data block. // Values are written as big endian. func uint32ToBlock(v0, v1 uint32, dst []byte) { dst[0] = byte(v0 >> 24) diff --git a/src/pkg/crypto/xtea/xtea_test.go b/src/pkg/crypto/xtea/xtea_test.go index 03934f169..217d96adc 100644 --- a/src/pkg/crypto/xtea/xtea_test.go +++ b/src/pkg/crypto/xtea/xtea_test.go @@ -8,7 +8,7 @@ import ( "testing" ) -// A sample test key for when we just want to initialise a cipher +// A sample test key for when we just want to initialize a cipher var testKey = []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF} // Test that the block size for XTEA is correct @@ -26,12 +26,12 @@ func TestBlocksize(t *testing.T) { result := c.BlockSize() if result != 8 { - t.Errorf("BlockSize function - expected 8, gotr %d", result) + t.Errorf("BlockSize function - expected 8, got %d", result) return } } -// A series of test values to confirm that the Cipher.table array was initialised correctly +// A series of test values to confirm that the Cipher.table array was initialized correctly var testTable = []uint32{ 0x00112233, 0x6B1568B8, 0xE28CE030, 0xC5089E2D, 0xC5089E2D, 0x1EFBD3A2, 0xA7845C2A, 0x78EF0917, 0x78EF0917, 0x172682D0, 0x5B6AC714, 0x822AC955, 0x3DE68511, 0xDC1DFECA, 0x2062430E, 0x3611343F, @@ -43,7 +43,7 @@ var testTable = []uint32{ 0x4E22726F, 0x309E306C, 0x309E306C, 0x8A9165E1, 0x1319EE69, 0xF595AC66, 0xF595AC66, 0x4F88E1DB, } -// Test that the cipher context is initialised correctly +// Test that the cipher context is initialized correctly func TestCipherInit(t *testing.T) { c, err := NewCipher(testKey) if err != nil { @@ -53,7 +53,7 @@ func TestCipherInit(t *testing.T) { for i := 0; i < len(c.table); i++ { if c.table[i] != testTable[i] { - t.Errorf("NewCipher() failed to initialise Cipher.table[%d] correctly. Expected %08X, got %08X", i, testTable[i], c.table[i]) + t.Errorf("NewCipher() failed to initialize Cipher.table[%d] correctly. Expected %08X, got %08X", i, testTable[i], c.table[i]) break } } diff --git a/src/pkg/debug/dwarf/type.go b/src/pkg/debug/dwarf/type.go index 902a545f8..f9acf119f 100644 --- a/src/pkg/debug/dwarf/type.go +++ b/src/pkg/debug/dwarf/type.go @@ -523,7 +523,7 @@ func (d *Data) Type(off Offset) (Type, os.Error) { // Attributes: // AttrType: type of return value if any // AttrName: possible name of type [ignored] - // AttrPrototyped: whether used ANSI C prototye [ignored] + // AttrPrototyped: whether used ANSI C prototype [ignored] // Children: // TagFormalParameter: typed parameter // AttrType: type of parameter diff --git a/src/pkg/debug/macho/file.go b/src/pkg/debug/macho/file.go index a777d873c..721a4c416 100644 --- a/src/pkg/debug/macho/file.go +++ b/src/pkg/debug/macho/file.go @@ -184,7 +184,7 @@ func (f *File) Close() os.Error { return err } -// NewFile creates a new File for acecssing a Mach-O binary in an underlying reader. +// NewFile creates a new File for accessing a Mach-O binary in an underlying reader. // The Mach-O binary is expected to start at position 0 in the ReaderAt. func NewFile(r io.ReaderAt) (*File, os.Error) { f := new(File) diff --git a/src/pkg/debug/pe/file.go b/src/pkg/debug/pe/file.go index 6a14e50f9..04991f781 100644 --- a/src/pkg/debug/pe/file.go +++ b/src/pkg/debug/pe/file.go @@ -112,7 +112,7 @@ func (f *File) Close() os.Error { return err } -// NewFile creates a new File for acecssing a PE binary in an underlying reader. +// NewFile creates a new File for accessing a PE binary in an underlying reader. func NewFile(r io.ReaderAt) (*File, os.Error) { f := new(File) sr := io.NewSectionReader(r, 0, 1<<63-1) diff --git a/src/pkg/debug/proc/proc_linux.go b/src/pkg/debug/proc/proc_linux.go index 17c8fa529..153c3e99b 100644 --- a/src/pkg/debug/proc/proc_linux.go +++ b/src/pkg/debug/proc/proc_linux.go @@ -277,7 +277,7 @@ func (t *thread) ptraceDetach() os.Error { } /* - * Logging utilties + * Logging utilities */ var logLock sync.Mutex @@ -1192,7 +1192,7 @@ func (p *process) attachAllThreads() os.Error { // We stop threads as we attach to them; however, because new // threads can appear while we're looping over all of them, we - // have to repeatly scan until we know we're attached to all + // have to repeatedly scan until we know we're attached to all // of them. for again := true; again; { again = false @@ -1214,7 +1214,7 @@ func (p *process) attachAllThreads() os.Error { _, err = p.attachThread(tid) if err != nil { // There could have been a race, or - // this process could be a zobmie. + // this process could be a zombie. statFile, err2 := ioutil.ReadFile(taskPath + "/" + tidStr + "/stat") if err2 != nil { switch err2 := err2.(type) { diff --git a/src/pkg/ebnf/ebnf.go b/src/pkg/ebnf/ebnf.go index 7918c4593..661afdd35 100644 --- a/src/pkg/ebnf/ebnf.go +++ b/src/pkg/ebnf/ebnf.go @@ -5,10 +5,10 @@ // Package ebnf is a library for EBNF grammars. The input is text ([]byte) // satisfying the following grammar (represented itself in EBNF): // -// Production = name "=" Expression "." . +// Production = name "=" [ Expression ] "." . // Expression = Alternative { "|" Alternative } . // Alternative = Term { Term } . -// Term = name | token [ "..." token ] | Group | Option | Repetition . +// Term = name | token [ "…" token ] | Group | Option | Repetition . // Group = "(" Expression ")" . // Option = "[" Expression "]" . // Repetition = "{" Expression "}" . @@ -82,6 +82,12 @@ type ( Body Expression // {body} } + // A Bad node stands for pieces of source code that lead to a parse error. + Bad struct { + TokPos token.Pos + Error string // parser error message + } + // A Production node represents an EBNF production. Production struct { Name *Name @@ -103,6 +109,7 @@ func (x *Range) Pos() token.Pos { return x.Begin.Pos() } func (x *Group) Pos() token.Pos { return x.Lparen } func (x *Option) Pos() token.Pos { return x.Lbrack } func (x *Repetition) Pos() token.Pos { return x.Lbrace } +func (x *Bad) Pos() token.Pos { return x.TokPos } func (x *Production) Pos() token.Pos { return x.Name.Pos() } diff --git a/src/pkg/ebnf/ebnf_test.go b/src/pkg/ebnf/ebnf_test.go index e77cf64ad..30301748d 100644 --- a/src/pkg/ebnf/ebnf_test.go +++ b/src/pkg/ebnf/ebnf_test.go @@ -14,7 +14,7 @@ import ( var fset = token.NewFileSet() -var grammars = []string{ +var goodGrammars = []string{ `Program = .`, `Program = foo . @@ -22,7 +22,7 @@ var grammars = []string{ `Program = "a" | "b" "c" .`, - `Program = "a" ... "z" .`, + `Program = "a" … "z" .`, `Program = Song . Song = { Note } . @@ -38,7 +38,19 @@ var grammars = []string{ } -func check(t *testing.T, filename string, src []byte) { +var badGrammars = []string{ + `Program = | .`, + `Program = | b .`, + `Program = a … b .`, + `Program = "a" … .`, + `Program = … "b" .`, + `Program = () .`, + `Program = [] .`, + `Program = {} .`, +} + + +func checkGood(t *testing.T, filename string, src []byte) { grammar, err := Parse(fset, filename, src) if err != nil { t.Errorf("Parse(%s) failed: %v", src, err) @@ -49,9 +61,20 @@ func check(t *testing.T, filename string, src []byte) { } +func checkBad(t *testing.T, filename string, src []byte) { + _, err := Parse(fset, filename, src) + if err == nil { + t.Errorf("Parse(%s) should have failed", src) + } +} + + func TestGrammars(t *testing.T) { - for _, src := range grammars { - check(t, "", []byte(src)) + for _, src := range goodGrammars { + checkGood(t, "", []byte(src)) + } + for _, src := range badGrammars { + checkBad(t, "", []byte(src)) } } @@ -67,6 +90,6 @@ func TestFiles(t *testing.T) { if err != nil { t.Fatal(err) } - check(t, filename, src) + checkGood(t, filename, src) } } diff --git a/src/pkg/ebnf/parser.go b/src/pkg/ebnf/parser.go index 818168e11..ede4f7073 100644 --- a/src/pkg/ebnf/parser.go +++ b/src/pkg/ebnf/parser.go @@ -85,6 +85,7 @@ func (p *parser) parseToken() *Token { } +// ParseTerm returns nil if no term was found. func (p *parser) parseTerm() (x Expression) { pos := p.pos @@ -95,7 +96,8 @@ func (p *parser) parseTerm() (x Expression) { case token.STRING: tok := p.parseToken() x = tok - if p.tok == token.ELLIPSIS { + const ellipsis = "…" // U+2026, the horizontal ellipsis character + if p.tok == token.ILLEGAL && p.lit == ellipsis { p.next() x = &Range{tok, p.parseToken()} } @@ -130,7 +132,8 @@ func (p *parser) parseSequence() Expression { // no need for a sequence if list.Len() < 2 switch len(list) { case 0: - return nil + p.errorExpected(p.pos, "term") + return &Bad{p.pos, "term expected"} case 1: return list[0] } @@ -143,20 +146,16 @@ func (p *parser) parseExpression() Expression { var list Alternative for { - if x := p.parseSequence(); x != nil { - list = append(list, x) - } + list = append(list, p.parseSequence()) if p.tok != token.OR { break } p.next() } + // len(list) > 0 // no need for an Alternative node if list.Len() < 2 - switch len(list) { - case 0: - return nil - case 1: + if len(list) == 1 { return list[0] } @@ -167,7 +166,10 @@ func (p *parser) parseExpression() Expression { func (p *parser) parseProduction() *Production { name := p.parseIdentifier() p.expect(token.ASSIGN) - expr := p.parseExpression() + var expr Expression + if p.tok != token.PERIOD { + expr = p.parseExpression() + } p.expect(token.PERIOD) return &Production{name, expr} } @@ -177,7 +179,7 @@ func (p *parser) parse(fset *token.FileSet, filename string, src []byte) Grammar // initialize parser p.fset = fset p.ErrorVector.Reset() - p.scanner.Init(fset.AddFile(filename, fset.Base(), len(src)), src, p, 0) + p.scanner.Init(fset.AddFile(filename, fset.Base(), len(src)), src, p, scanner.AllowIllegalChars) p.next() // initializes pos, tok, lit grammar := make(Grammar) diff --git a/src/pkg/encoding/base64/base64.go b/src/pkg/encoding/base64/base64.go index 496129798..c6b2a13e4 100644 --- a/src/pkg/encoding/base64/base64.go +++ b/src/pkg/encoding/base64/base64.go @@ -106,6 +106,13 @@ func (enc *Encoding) Encode(dst, src []byte) { } } +// EncodeToString returns the base64 encoding of src. +func (enc *Encoding) EncodeToString(src []byte) string { + buf := make([]byte, enc.EncodedLen(len(src))) + enc.Encode(buf, src) + return string(buf) +} + type encoder struct { err os.Error enc *Encoding @@ -260,6 +267,13 @@ func (enc *Encoding) Decode(dst, src []byte) (n int, err os.Error) { return } +// DecodeString returns the bytes represented by the base64 string s. +func (enc *Encoding) DecodeString(s string) ([]byte, os.Error) { + dbuf := make([]byte, enc.DecodedLen(len(s))) + n, err := enc.Decode(dbuf, []byte(s)) + return dbuf[:n], err +} + type decoder struct { err os.Error enc *Encoding diff --git a/src/pkg/encoding/base64/base64_test.go b/src/pkg/encoding/base64/base64_test.go index de41e704b..c163dae84 100644 --- a/src/pkg/encoding/base64/base64_test.go +++ b/src/pkg/encoding/base64/base64_test.go @@ -56,9 +56,8 @@ func testEqual(t *testing.T, msg string, args ...interface{}) bool { func TestEncode(t *testing.T) { for _, p := range pairs { - buf := make([]byte, StdEncoding.EncodedLen(len(p.decoded))) - StdEncoding.Encode(buf, []byte(p.decoded)) - testEqual(t, "Encode(%q) = %q, want %q", p.decoded, string(buf), p.encoded) + got := StdEncoding.EncodeToString([]byte(p.decoded)) + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, got, p.encoded) } } @@ -102,6 +101,10 @@ func TestDecode(t *testing.T) { testEqual(t, "Decode(%q) = end %v, want %v", p.encoded, end, (p.encoded[len(p.encoded)-1] == '=')) } testEqual(t, "Decode(%q) = %q, want %q", p.encoded, string(dbuf[0:count]), p.decoded) + + dbuf, err = StdEncoding.DecodeString(p.encoded) + testEqual(t, "DecodeString(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "DecodeString(%q) = %q, want %q", string(dbuf), p.decoded) } } diff --git a/src/pkg/encoding/binary/binary.go b/src/pkg/encoding/binary/binary.go index a01d0e024..8e55cb23b 100644 --- a/src/pkg/encoding/binary/binary.go +++ b/src/pkg/encoding/binary/binary.go @@ -125,6 +125,35 @@ func (bigEndian) GoString() string { return "binary.BigEndian" } // Bytes read from r are decoded using the specified byte order // and written to successive fields of the data. func Read(r io.Reader, order ByteOrder, data interface{}) os.Error { + // Fast path for basic types. + if n := intDestSize(data); n != 0 { + var b [8]byte + bs := b[:n] + if _, err := io.ReadFull(r, bs); err != nil { + return err + } + switch v := data.(type) { + case *int8: + *v = int8(b[0]) + case *uint8: + *v = b[0] + case *int16: + *v = int16(order.Uint16(bs)) + case *uint16: + *v = order.Uint16(bs) + case *int32: + *v = int32(order.Uint32(bs)) + case *uint32: + *v = order.Uint32(bs) + case *int64: + *v = int64(order.Uint64(bs)) + case *uint64: + *v = order.Uint64(bs) + } + return nil + } + + // Fallback to reflect-based. var v reflect.Value switch d := reflect.ValueOf(data); d.Kind() { case reflect.Ptr: @@ -155,6 +184,63 @@ func Read(r io.Reader, order ByteOrder, data interface{}) os.Error { // Bytes written to w are encoded using the specified byte order // and read from successive fields of the data. func Write(w io.Writer, order ByteOrder, data interface{}) os.Error { + // Fast path for basic types. + var b [8]byte + var bs []byte + switch v := data.(type) { + case *int8: + bs = b[:1] + b[0] = byte(*v) + case int8: + bs = b[:1] + b[0] = byte(v) + case *uint8: + bs = b[:1] + b[0] = *v + case uint8: + bs = b[:1] + b[0] = byte(v) + case *int16: + bs = b[:2] + order.PutUint16(bs, uint16(*v)) + case int16: + bs = b[:2] + order.PutUint16(bs, uint16(v)) + case *uint16: + bs = b[:2] + order.PutUint16(bs, *v) + case uint16: + bs = b[:2] + order.PutUint16(bs, v) + case *int32: + bs = b[:4] + order.PutUint32(bs, uint32(*v)) + case int32: + bs = b[:4] + order.PutUint32(bs, uint32(v)) + case *uint32: + bs = b[:4] + order.PutUint32(bs, *v) + case uint32: + bs = b[:4] + order.PutUint32(bs, v) + case *int64: + bs = b[:8] + order.PutUint64(bs, uint64(*v)) + case int64: + bs = b[:8] + order.PutUint64(bs, uint64(v)) + case *uint64: + bs = b[:8] + order.PutUint64(bs, *v) + case uint64: + bs = b[:8] + order.PutUint64(bs, v) + } + if bs != nil { + _, err := w.Write(bs) + return err + } v := reflect.Indirect(reflect.ValueOf(data)) size := TotalSize(v) if size < 0 { @@ -394,3 +480,19 @@ func (e *encoder) value(v reflect.Value) { } } } + +// intDestSize returns the size of the integer that ptrType points to, +// or 0 if the type is not supported. +func intDestSize(ptrType interface{}) int { + switch ptrType.(type) { + case *int8, *uint8: + return 1 + case *int16, *uint16: + return 2 + case *int32, *uint32: + return 4 + case *int64, *uint64: + return 8 + } + return 0 +} diff --git a/src/pkg/encoding/binary/binary_test.go b/src/pkg/encoding/binary/binary_test.go index 7857c68d3..b266996f6 100644 --- a/src/pkg/encoding/binary/binary_test.go +++ b/src/pkg/encoding/binary/binary_test.go @@ -5,6 +5,7 @@ package binary import ( + "io" "os" "bytes" "math" @@ -160,3 +161,75 @@ func TestWriteT(t *testing.T) { } } } + +type byteSliceReader struct { + remain []byte +} + +func (br *byteSliceReader) Read(p []byte) (int, os.Error) { + n := copy(p, br.remain) + br.remain = br.remain[n:] + return n, nil +} + +func BenchmarkRead(b *testing.B) { + var ls Struct + bsr := &byteSliceReader{} + var r io.Reader = bsr + + for i := 0; i < b.N; i++ { + bsr.remain = big + Read(r, BigEndian, &ls.Int8) + Read(r, BigEndian, &ls.Int16) + Read(r, BigEndian, &ls.Int32) + Read(r, BigEndian, &ls.Int64) + Read(r, BigEndian, &ls.Uint8) + Read(r, BigEndian, &ls.Uint16) + Read(r, BigEndian, &ls.Uint32) + Read(r, BigEndian, &ls.Uint64) + } + + want := s + want.Float32 = 0 + want.Float64 = 0 + want.Complex64 = 0 + want.Complex128 = 0 + for i := range want.Array { + want.Array[i] = 0 + } + if !reflect.DeepEqual(ls, want) { + panic("no match") + } +} + +func BenchmarkWrite(b *testing.B) { + buf := new(bytes.Buffer) + var w io.Writer = buf + + for i := 0; i < b.N; i++ { + buf.Reset() + Write(w, BigEndian, &s.Int8) + Write(w, BigEndian, &s.Int16) + Write(w, BigEndian, &s.Int32) + Write(w, BigEndian, &s.Int64) + Write(w, BigEndian, &s.Uint8) + Write(w, BigEndian, &s.Uint16) + Write(w, BigEndian, &s.Uint32) + Write(w, BigEndian, &s.Uint64) + Write(w, BigEndian, s.Int8) + Write(w, BigEndian, s.Int16) + Write(w, BigEndian, s.Int32) + Write(w, BigEndian, s.Int64) + Write(w, BigEndian, s.Uint8) + Write(w, BigEndian, s.Uint16) + Write(w, BigEndian, s.Uint32) + Write(w, BigEndian, s.Uint64) + } + + if !bytes.Equal(buf.Bytes()[:30], big[:30]) { + panic("first half doesn't match") + } + if !bytes.Equal(buf.Bytes()[30:], big[:30]) { + panic("second half doesn't match") + } +} diff --git a/src/pkg/encoding/git85/git.go b/src/pkg/encoding/git85/git.go index 09a45cd3c..6bb74f46c 100644 --- a/src/pkg/encoding/git85/git.go +++ b/src/pkg/encoding/git85/git.go @@ -273,5 +273,5 @@ func (d *decoder) Read(p []byte) (n int, err os.Error) { d.nbuf = copy(d.buf[0:], d.buf[nl+1:d.nbuf]) d.off += int64(nl + 1) } - panic("unreacahable") + panic("unreachable") } diff --git a/src/pkg/encoding/hex/hex.go b/src/pkg/encoding/hex/hex.go index 891de1861..47cdedd60 100644 --- a/src/pkg/encoding/hex/hex.go +++ b/src/pkg/encoding/hex/hex.go @@ -6,6 +6,8 @@ package hex import ( + "bytes" + "io" "os" "strconv" ) @@ -99,3 +101,117 @@ func DecodeString(s string) ([]byte, os.Error) { } return dst, nil } + +// Dump returns a string that contains a hex dump of the given data. The format +// of the hex dump matches the output of `hexdump -C` on the command line. +func Dump(data []byte) string { + buf := bytes.NewBuffer(nil) + dumper := Dumper(buf) + dumper.Write(data) + dumper.Close() + return string(buf.Bytes()) +} + +// Dumper returns a WriteCloser that writes a hex dump of all written data to +// w. The format of the dump matches the output of `hexdump -C` on the command +// line. +func Dumper(w io.Writer) io.WriteCloser { + return &dumper{w: w} +} + +type dumper struct { + w io.Writer + rightChars [18]byte + buf [14]byte + used int // number of bytes in the current line + n uint // number of bytes, total +} + +func toChar(b byte) byte { + if b < 32 || b > 126 { + return '.' + } + return b +} + +func (h *dumper) Write(data []byte) (n int, err os.Error) { + // Output lines look like: + // 00000010 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d |./0123456789:;<=| + // ^ offset ^ extra space ^ ASCII of line. + for i := range data { + if h.used == 0 { + // At the beginning of a line we print the current + // offset in hex. + h.buf[0] = byte(h.n >> 24) + h.buf[1] = byte(h.n >> 16) + h.buf[2] = byte(h.n >> 8) + h.buf[3] = byte(h.n) + Encode(h.buf[4:], h.buf[:4]) + h.buf[12] = ' ' + h.buf[13] = ' ' + _, err = h.w.Write(h.buf[4:]) + } + Encode(h.buf[:], data[i:i+1]) + h.buf[2] = ' ' + l := 3 + if h.used == 7 { + // There's an additional space after the 8th byte. + h.buf[3] = ' ' + l = 4 + } else if h.used == 15 { + // At the end of the line there's an extra space and + // the bar for the right column. + h.buf[3] = ' ' + h.buf[4] = '|' + l = 5 + } + _, err = h.w.Write(h.buf[:l]) + if err != nil { + return + } + n++ + h.rightChars[h.used] = toChar(data[i]) + h.used++ + h.n++ + if h.used == 16 { + h.rightChars[16] = '|' + h.rightChars[17] = '\n' + _, err = h.w.Write(h.rightChars[:]) + if err != nil { + return + } + h.used = 0 + } + } + return +} + +func (h *dumper) Close() (err os.Error) { + // See the comments in Write() for the details of this format. + if h.used == 0 { + return + } + h.buf[0] = ' ' + h.buf[1] = ' ' + h.buf[2] = ' ' + h.buf[3] = ' ' + h.buf[4] = '|' + nBytes := h.used + for h.used < 16 { + l := 3 + if h.used == 7 { + l = 4 + } else if h.used == 15 { + l = 5 + } + _, err = h.w.Write(h.buf[:l]) + if err != nil { + return + } + h.used++ + } + h.rightChars[nBytes] = '|' + h.rightChars[nBytes+1] = '\n' + _, err = h.w.Write(h.rightChars[:nBytes+2]) + return +} diff --git a/src/pkg/encoding/hex/hex_test.go b/src/pkg/encoding/hex/hex_test.go index a14c9d4f4..8e1838e51 100644 --- a/src/pkg/encoding/hex/hex_test.go +++ b/src/pkg/encoding/hex/hex_test.go @@ -147,3 +147,46 @@ func TestDecodeString(t *testing.T) { } } } + +func TestDumper(t *testing.T) { + var in [40]byte + for i := range in { + in[i] = byte(i + 30) + } + + for stride := 1; stride < len(in); stride++ { + out := bytes.NewBuffer(nil) + dumper := Dumper(out) + done := 0 + for done < len(in) { + todo := done + stride + if todo > len(in) { + todo = len(in) + } + dumper.Write(in[done:todo]) + done = todo + } + + dumper.Close() + if !bytes.Equal(out.Bytes(), expectedHexDump) { + t.Errorf("stride: %d failed. got:\n%s\nwant:\n%s", stride, out.Bytes(), expectedHexDump) + } + } +} + +func TestDump(t *testing.T) { + var in [40]byte + for i := range in { + in[i] = byte(i + 30) + } + + out := []byte(Dump(in[:])) + if !bytes.Equal(out, expectedHexDump) { + t.Errorf("got:\n%s\nwant:\n%s", out, expectedHexDump) + } +} + +var expectedHexDump = []byte(`00000000 1e 1f 20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d |.. !"#$%&'()*+,-| +00000010 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d |./0123456789:;<=| +00000020 3e 3f 40 41 42 43 44 45 |>?@ABCDE| +`) diff --git a/src/pkg/encoding/line/line.go b/src/pkg/encoding/line/line.go deleted file mode 100644 index 123962b1f..000000000 --- a/src/pkg/encoding/line/line.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package line implements a Reader that reads lines delimited by '\n' or -// ' \r\n'. -package line - -import ( - "io" - "os" -) - -// Reader reads lines, delimited by '\n' or \r\n', from an io.Reader. -type Reader struct { - buf []byte - consumed int - in io.Reader - err os.Error -} - -// NewReader returns a new Reader that will read successive -// lines from the input Reader. -func NewReader(input io.Reader, maxLineLength int) *Reader { - return &Reader{ - buf: make([]byte, 0, maxLineLength), - consumed: 0, - in: input, - } -} - -// Read reads from any buffered data past the last line read, or from the underlying -// io.Reader if the buffer is empty. -func (l *Reader) Read(p []byte) (n int, err os.Error) { - l.removeConsumedFromBuffer() - if len(l.buf) > 0 { - n = copy(p, l.buf) - l.consumed += n - return - } - return l.in.Read(p) -} - -func (l *Reader) removeConsumedFromBuffer() { - if l.consumed > 0 { - n := copy(l.buf, l.buf[l.consumed:]) - l.buf = l.buf[:n] - l.consumed = 0 - } -} - -// ReadLine tries to return a single line, not including the end-of-line bytes. -// If the line was found to be longer than the maximum length then isPrefix is -// set and the beginning of the line is returned. The rest of the line will be -// returned from future calls. isPrefix will be false when returning the last -// fragment of the line. The returned buffer points into the internal state of -// the Reader and is only valid until the next call to ReadLine. ReadLine -// either returns a non-nil line or it returns an error, never both. -func (l *Reader) ReadLine() (line []byte, isPrefix bool, err os.Error) { - l.removeConsumedFromBuffer() - - if len(l.buf) == 0 && l.err != nil { - err = l.err - return - } - - scannedTo := 0 - - for { - i := scannedTo - for ; i < len(l.buf); i++ { - if l.buf[i] == '\r' && len(l.buf) > i+1 && l.buf[i+1] == '\n' { - line = l.buf[:i] - l.consumed = i + 2 - return - } else if l.buf[i] == '\n' { - line = l.buf[:i] - l.consumed = i + 1 - return - } - } - - if i == cap(l.buf) { - line = l.buf[:i] - l.consumed = i - isPrefix = true - return - } - - if l.err != nil { - line = l.buf - l.consumed = i - return - } - - // We don't want to rescan the input that we just scanned. - // However, we need to back up one byte because the last byte - // could have been a '\r' and we do need to rescan that. - scannedTo = i - if scannedTo > 0 { - scannedTo-- - } - oldLen := len(l.buf) - l.buf = l.buf[:cap(l.buf)] - n, readErr := l.in.Read(l.buf[oldLen:]) - l.buf = l.buf[:oldLen+n] - if readErr != nil { - l.err = readErr - if len(l.buf) == 0 { - return nil, false, readErr - } - } - } - panic("unreachable") -} diff --git a/src/pkg/encoding/line/line_test.go b/src/pkg/encoding/line/line_test.go deleted file mode 100644 index ff3d51669..000000000 --- a/src/pkg/encoding/line/line_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package line - -import ( - "bytes" - "io" - "io/ioutil" - "os" - "testing" -) - -var testOutput = []byte("0123456789abcdefghijklmnopqrstuvwxy") -var testInput = []byte("012\n345\n678\n9ab\ncde\nfgh\nijk\nlmn\nopq\nrst\nuvw\nxy") -var testInputrn = []byte("012\r\n345\r\n678\r\n9ab\r\ncde\r\nfgh\r\nijk\r\nlmn\r\nopq\r\nrst\r\nuvw\r\nxy\r\n\n\r\n") - -// TestReader wraps a []byte and returns reads of a specific length. -type testReader struct { - data []byte - stride int -} - -func (t *testReader) Read(buf []byte) (n int, err os.Error) { - n = t.stride - if n > len(t.data) { - n = len(t.data) - } - if n > len(buf) { - n = len(buf) - } - copy(buf, t.data) - t.data = t.data[n:] - if len(t.data) == 0 { - err = os.EOF - } - return -} - -func testLineReader(t *testing.T, input []byte) { - for stride := 1; stride < len(input); stride++ { - done := 0 - reader := testReader{input, stride} - l := NewReader(&reader, len(input)+1) - for { - line, isPrefix, err := l.ReadLine() - if len(line) > 0 && err != nil { - t.Errorf("ReadLine returned both data and error: %s", err) - } - if isPrefix { - t.Errorf("ReadLine returned prefix") - } - if err != nil { - if err != os.EOF { - t.Fatalf("Got unknown error: %s", err) - } - break - } - if want := testOutput[done : done+len(line)]; !bytes.Equal(want, line) { - t.Errorf("Bad line at stride %d: want: %x got: %x", stride, want, line) - } - done += len(line) - } - if done != len(testOutput) { - t.Error("ReadLine didn't return everything") - } - } -} - -func TestReader(t *testing.T) { - testLineReader(t, testInput) - testLineReader(t, testInputrn) -} - -func TestLineTooLong(t *testing.T) { - buf := bytes.NewBuffer([]byte("aaabbbcc\n")) - l := NewReader(buf, 3) - line, isPrefix, err := l.ReadLine() - if !isPrefix || !bytes.Equal(line, []byte("aaa")) || err != nil { - t.Errorf("bad result for first line: %x %s", line, err) - } - line, isPrefix, err = l.ReadLine() - if !isPrefix || !bytes.Equal(line, []byte("bbb")) || err != nil { - t.Errorf("bad result for second line: %x", line) - } - line, isPrefix, err = l.ReadLine() - if isPrefix || !bytes.Equal(line, []byte("cc")) || err != nil { - t.Errorf("bad result for third line: %x", line) - } -} - -func TestReadAfterLines(t *testing.T) { - line1 := "line1" - restData := "line2\nline 3\n" - inbuf := bytes.NewBuffer([]byte(line1 + "\n" + restData)) - outbuf := new(bytes.Buffer) - maxLineLength := len(line1) + len(restData)/2 - l := NewReader(inbuf, maxLineLength) - line, isPrefix, err := l.ReadLine() - if isPrefix || err != nil || string(line) != line1 { - t.Errorf("bad result for first line: isPrefix=%v err=%v line=%q", isPrefix, err, string(line)) - } - n, err := io.Copy(outbuf, l) - if int(n) != len(restData) || err != nil { - t.Errorf("bad result for Read: n=%d err=%v", n, err) - } - if outbuf.String() != restData { - t.Errorf("bad result for Read: got %q; expected %q", outbuf.String(), restData) - } -} - -func TestReadEmptyBuffer(t *testing.T) { - l := NewReader(bytes.NewBuffer(nil), 10) - line, isPrefix, err := l.ReadLine() - if err != os.EOF { - t.Errorf("expected EOF from ReadLine, got '%s' %t %s", line, isPrefix, err) - } -} - -func TestLinesAfterRead(t *testing.T) { - l := NewReader(bytes.NewBuffer([]byte("foo")), 10) - _, err := ioutil.ReadAll(l) - if err != nil { - t.Error(err) - return - } - - line, isPrefix, err := l.ReadLine() - if err != os.EOF { - t.Errorf("expected EOF from ReadLine, got '%s' %t %s", line, isPrefix, err) - } -} diff --git a/src/pkg/encoding/pem/pem.go b/src/pkg/encoding/pem/pem.go index 44e3d0ad0..c2398807f 100644 --- a/src/pkg/encoding/pem/pem.go +++ b/src/pkg/encoding/pem/pem.go @@ -140,7 +140,7 @@ Error: // any lines which could be header lines. However, a valid preamble // line is not a valid header line, therefore we cannot have consumed // the preamble line for the any subsequent block. Thus, we will always - // find any valid block, no matter what bytes preceed it. + // find any valid block, no matter what bytes precede it. // // For example, if the input is // diff --git a/src/pkg/exec/exec.go b/src/pkg/exec/exec.go index 043f84728..935f24c21 100644 --- a/src/pkg/exec/exec.go +++ b/src/pkg/exec/exec.go @@ -7,198 +7,363 @@ // adjustments. package exec -// BUG(r): This package should be made even easier to use or merged into os. - import ( + "bytes" + "io" "os" "strconv" ) -// Arguments to Run. -const ( - DevNull = iota - PassThrough - Pipe - MergeWithStdout -) +// Error records the name of a binary that failed to be be executed +// and the reason it failed. +type Error struct { + Name string + Error os.Error +} -// A Cmd represents a running command. -// Stdin, Stdout, and Stderr are Files representing pipes -// connected to the running command's standard input, output, and error, -// or else nil, depending on the arguments to Run. -// Process represents the underlying operating system process. +func (e *Error) String() string { + return "exec: " + strconv.Quote(e.Name) + ": " + e.Error.String() +} + +// Cmd represents an external command being prepared or run. type Cmd struct { - Stdin *os.File - Stdout *os.File - Stderr *os.File + // Path is the path of the command to run. + // + // This is the only field that must be set to a non-zero + // value. + Path string + + // Args holds command line arguments, including the command as Args[0]. + // If the Args field is empty or nil, Run uses {Path}. + // + // In typical use, both Path and Args are set by calling Command. + Args []string + + // Env specifies the environment of the process. + // If Env is nil, Run uses the current process's environment. + Env []string + + // Dir specifies the working directory of the command. + // If Dir is the empty string, Run runs the command in the + // calling process's current directory. + Dir string + + // Stdin specifies the process's standard input. + // If Stdin is nil, the process reads from DevNull. + Stdin io.Reader + + // Stdout and Stderr specify the process's standard output and error. + // + // If either is nil, Run connects the + // corresponding file descriptor to /dev/null. + // + // If Stdout and Stderr are are the same writer, at most one + // goroutine at a time will call Write. + Stdout io.Writer + Stderr io.Writer + + // Process is the underlying process, once started. Process *os.Process + + err os.Error // last error (from LookPath, stdin, stdout, stderr) + finished bool // when Wait was called + childFiles []*os.File + closeAfterStart []io.Closer + closeAfterWait []io.Closer + goroutine []func() os.Error + errch chan os.Error // one send per goroutine } -// PathError records the name of a binary that was not -// found on the current $PATH. -type PathError struct { - Name string +// Command returns the Cmd struct to execute the named program with +// the given arguments. +// +// It sets Path and Args in the returned structure and zeroes the +// other fields. +// +// If name contains no path separators, Command uses LookPath to +// resolve the path to a complete name if possible. Otherwise it uses +// name directly. +// +// The returned Cmd's Args field is constructed from the command name +// followed by the elements of arg, so arg should not include the +// command name itself. For example, Command("echo", "hello") +func Command(name string, arg ...string) *Cmd { + aname, err := LookPath(name) + if err != nil { + aname = name + } + return &Cmd{ + Path: aname, + Args: append([]string{name}, arg...), + err: err, + } } -func (e *PathError) String() string { - return "command " + strconv.Quote(e.Name) + " not found in $PATH" +// interfaceEqual protects against panics from doing equality tests on +// two interfaces with non-comparable underlying types +func interfaceEqual(a, b interface{}) bool { + defer func() { + recover() + }() + return a == b } -// Given mode (DevNull, etc), return file for child -// and file to record in Cmd structure. -func modeToFiles(mode, fd int) (*os.File, *os.File, os.Error) { - switch mode { - case DevNull: - rw := os.O_WRONLY - if fd == 0 { - rw = os.O_RDONLY - } - f, err := os.OpenFile(os.DevNull, rw, 0) - return f, nil, err - case PassThrough: - switch fd { - case 0: - return os.Stdin, nil, nil - case 1: - return os.Stdout, nil, nil - case 2: - return os.Stderr, nil, nil - } - case Pipe: - r, w, err := os.Pipe() - if err != nil { - return nil, nil, err - } - if fd == 0 { - return r, w, nil +func (c *Cmd) envv() []string { + if c.Env != nil { + return c.Env + } + return os.Environ() +} + +func (c *Cmd) argv() []string { + if len(c.Args) > 0 { + return c.Args + } + return []string{c.Path} +} + +func (c *Cmd) stdin() (f *os.File, err os.Error) { + if c.Stdin == nil { + f, err = os.Open(os.DevNull) + c.closeAfterStart = append(c.closeAfterStart, f) + return + } + + if f, ok := c.Stdin.(*os.File); ok { + return f, nil + } + + pr, pw, err := os.Pipe() + if err != nil { + return + } + + c.closeAfterStart = append(c.closeAfterStart, pr) + c.closeAfterWait = append(c.closeAfterWait, pw) + c.goroutine = append(c.goroutine, func() os.Error { + _, err := io.Copy(pw, c.Stdin) + if err1 := pw.Close(); err == nil { + err = err1 } - return w, r, nil + return err + }) + return pr, nil +} + +func (c *Cmd) stdout() (f *os.File, err os.Error) { + return c.writerDescriptor(c.Stdout) +} + +func (c *Cmd) stderr() (f *os.File, err os.Error) { + if c.Stderr != nil && interfaceEqual(c.Stderr, c.Stdout) { + return c.childFiles[1], nil } - return nil, nil, os.EINVAL + return c.writerDescriptor(c.Stderr) } -// Run starts the named binary running with -// arguments argv and environment envv. -// If the dir argument is not empty, the child changes -// into the directory before executing the binary. -// It returns a pointer to a new Cmd representing -// the command or an error. +func (c *Cmd) writerDescriptor(w io.Writer) (f *os.File, err os.Error) { + if w == nil { + f, err = os.OpenFile(os.DevNull, os.O_WRONLY, 0) + c.closeAfterStart = append(c.closeAfterStart, f) + return + } + + if f, ok := w.(*os.File); ok { + return f, nil + } + + pr, pw, err := os.Pipe() + if err != nil { + return + } + + c.closeAfterStart = append(c.closeAfterStart, pw) + c.closeAfterWait = append(c.closeAfterWait, pr) + c.goroutine = append(c.goroutine, func() os.Error { + _, err := io.Copy(w, pr) + return err + }) + return pw, nil +} + +// Run starts the specified command and waits for it to complete. +// +// The returned error is nil if the command runs, has no problems +// copying stdin, stdout, and stderr, and exits with a zero exit +// status. // -// The arguments stdin, stdout, and stderr -// specify how to handle standard input, output, and error. -// The choices are DevNull (connect to /dev/null), -// PassThrough (connect to the current process's standard stream), -// Pipe (connect to an operating system pipe), and -// MergeWithStdout (only for standard error; use the same -// file descriptor as was used for standard output). -// If an argument is Pipe, then the corresponding field (Stdin, Stdout, Stderr) -// of the returned Cmd is the other end of the pipe. -// Otherwise the field in Cmd is nil. -func Run(name string, argv, envv []string, dir string, stdin, stdout, stderr int) (c *Cmd, err os.Error) { - c = new(Cmd) - var fd [3]*os.File - - if fd[0], c.Stdin, err = modeToFiles(stdin, 0); err != nil { - goto Error - } - if fd[1], c.Stdout, err = modeToFiles(stdout, 1); err != nil { - goto Error - } - if stderr == MergeWithStdout { - fd[2] = fd[1] - } else if fd[2], c.Stderr, err = modeToFiles(stderr, 2); err != nil { - goto Error - } - - // Run command. - c.Process, err = os.StartProcess(name, argv, &os.ProcAttr{Dir: dir, Files: fd[:], Env: envv}) +// If the command fails to run or doesn't complete successfully, the +// error is of type *os.Waitmsg. Other error types may be +// returned for I/O problems. +func (c *Cmd) Run() os.Error { + if err := c.Start(); err != nil { + return err + } + return c.Wait() +} + +// Start starts the specified command but does not wait for it to complete. +func (c *Cmd) Start() os.Error { + if c.err != nil { + return c.err + } + if c.Process != nil { + return os.NewError("exec: already started") + } + + type F func(*Cmd) (*os.File, os.Error) + for _, setupFd := range []F{(*Cmd).stdin, (*Cmd).stdout, (*Cmd).stderr} { + fd, err := setupFd(c) + if err != nil { + return err + } + c.childFiles = append(c.childFiles, fd) + } + + var err os.Error + c.Process, err = os.StartProcess(c.Path, c.argv(), &os.ProcAttr{ + Dir: c.Dir, + Files: c.childFiles, + Env: c.envv(), + }) if err != nil { - goto Error + return err + } + + for _, fd := range c.closeAfterStart { + fd.Close() } - if fd[0] != os.Stdin { - fd[0].Close() + + c.errch = make(chan os.Error, len(c.goroutine)) + for _, fn := range c.goroutine { + go func(fn func() os.Error) { + c.errch <- fn() + }(fn) } - if fd[1] != os.Stdout { - fd[1].Close() + + return nil +} + +// Wait waits for the command to exit. +// It must have been started by Start. +// +// The returned error is nil if the command runs, has no problems +// copying stdin, stdout, and stderr, and exits with a zero exit +// status. +// +// If the command fails to run or doesn't complete successfully, the +// error is of type *os.Waitmsg. Other error types may be +// returned for I/O problems. +func (c *Cmd) Wait() os.Error { + if c.Process == nil { + return os.NewError("exec: not started") } - if fd[2] != os.Stderr && fd[2] != fd[1] { - fd[2].Close() + if c.finished { + return os.NewError("exec: Wait was already called") } - return c, nil + c.finished = true + msg, err := c.Process.Wait(0) -Error: - if fd[0] != os.Stdin && fd[0] != nil { - fd[0].Close() + var copyError os.Error + for _ = range c.goroutine { + if err := <-c.errch; err != nil && copyError == nil { + copyError = err + } } - if fd[1] != os.Stdout && fd[1] != nil { - fd[1].Close() + + for _, fd := range c.closeAfterWait { + fd.Close() } - if fd[2] != os.Stderr && fd[2] != nil && fd[2] != fd[1] { - fd[2].Close() + + if err != nil { + return err + } else if !msg.Exited() || msg.ExitStatus() != 0 { + return msg } - if c.Stdin != nil { - c.Stdin.Close() + + return copyError +} + +// Output runs the command and returns its standard output. +func (c *Cmd) Output() ([]byte, os.Error) { + if c.Stdout != nil { + return nil, os.NewError("exec: Stdout already set") } + var b bytes.Buffer + c.Stdout = &b + err := c.Run() + return b.Bytes(), err +} + +// CombinedOutput runs the command and returns its combined standard +// output and standard error. +func (c *Cmd) CombinedOutput() ([]byte, os.Error) { if c.Stdout != nil { - c.Stdout.Close() + return nil, os.NewError("exec: Stdout already set") } if c.Stderr != nil { - c.Stderr.Close() + return nil, os.NewError("exec: Stderr already set") } - if c.Process != nil { - c.Process.Release() - } - return nil, err + var b bytes.Buffer + c.Stdout = &b + c.Stderr = &b + err := c.Run() + return b.Bytes(), err } -// Wait waits for the running command c, -// returning the Waitmsg returned when the process exits. -// The options are passed to the process's Wait method. -// Setting options to 0 waits for c to exit; -// other options cause Wait to return for other -// process events; see package os for details. -func (c *Cmd) Wait(options int) (*os.Waitmsg, os.Error) { - if c.Process == nil { - return nil, os.ErrorString("exec: invalid use of Cmd.Wait") +// StdinPipe returns a pipe that will be connected to the command's +// standard input when the command starts. +func (c *Cmd) StdinPipe() (io.WriteCloser, os.Error) { + if c.Stdin != nil { + return nil, os.NewError("exec: Stdin already set") } - w, err := c.Process.Wait(options) - if w != nil && (w.Exited() || w.Signaled()) { - c.Process.Release() - c.Process = nil + if c.Process != nil { + return nil, os.NewError("exec: StdinPipe after process started") + } + pr, pw, err := os.Pipe() + if err != nil { + return nil, err } - return w, err + c.Stdin = pr + c.closeAfterStart = append(c.closeAfterStart, pr) + c.closeAfterWait = append(c.closeAfterStart, pw) + return pw, nil } -// Close waits for the running command c to exit, -// if it hasn't already, and then closes the non-nil file descriptors -// c.Stdin, c.Stdout, and c.Stderr. -func (c *Cmd) Close() os.Error { +// StdoutPipe returns a pipe that will be connected to the command's +// standard output when the command starts. +func (c *Cmd) StdoutPipe() (io.Reader, os.Error) { + if c.Stdout != nil { + return nil, os.NewError("exec: Stdout already set") + } if c.Process != nil { - // Loop on interrupt, but - // ignore other errors -- maybe - // caller has already waited for pid. - _, err := c.Wait(0) - for err == os.EINTR { - _, err = c.Wait(0) - } + return nil, os.NewError("exec: StdoutPipe after process started") } + pr, pw, err := os.Pipe() + if err != nil { + return nil, err + } + c.Stdout = pw + c.closeAfterStart = append(c.closeAfterStart, pw) + c.closeAfterWait = append(c.closeAfterStart, pr) + return pr, nil +} - // Close the FDs that are still open. - var err os.Error - if c.Stdin != nil && c.Stdin.Fd() >= 0 { - if err1 := c.Stdin.Close(); err1 != nil { - err = err1 - } +// StderrPipe returns a pipe that will be connected to the command's +// standard error when the command starts. +func (c *Cmd) StderrPipe() (io.Reader, os.Error) { + if c.Stderr != nil { + return nil, os.NewError("exec: Stderr already set") } - if c.Stdout != nil && c.Stdout.Fd() >= 0 { - if err1 := c.Stdout.Close(); err1 != nil && err != nil { - err = err1 - } + if c.Process != nil { + return nil, os.NewError("exec: StderrPipe after process started") } - if c.Stderr != nil && c.Stderr != c.Stdout && c.Stderr.Fd() >= 0 { - if err1 := c.Stderr.Close(); err1 != nil && err != nil { - err = err1 - } + pr, pw, err := os.Pipe() + if err != nil { + return nil, err } - return err + c.Stderr = pw + c.closeAfterStart = append(c.closeAfterStart, pw) + c.closeAfterWait = append(c.closeAfterStart, pr) + return pr, nil } diff --git a/src/pkg/exec/exec_test.go b/src/pkg/exec/exec_test.go index eb8cd5fec..c45a7d70a 100644 --- a/src/pkg/exec/exec_test.go +++ b/src/pkg/exec/exec_test.go @@ -5,163 +5,211 @@ package exec import ( + "bufio" + "bytes" + "fmt" "io" - "io/ioutil" "testing" "os" + "strconv" + "strings" ) -func run(argv []string, stdin, stdout, stderr int) (p *Cmd, err os.Error) { - exe, err := LookPath(argv[0]) - if err != nil { - return nil, err - } - return Run(exe, argv, nil, "", stdin, stdout, stderr) +func helperCommand(s ...string) *Cmd { + cs := []string{"-test.run=exec.TestHelperProcess", "--"} + cs = append(cs, s...) + cmd := Command(os.Args[0], cs...) + cmd.Env = append([]string{"GO_WANT_HELPER_PROCESS=1"}, os.Environ()...) + return cmd } -func TestRunCat(t *testing.T) { - cmd, err := run([]string{"cat"}, Pipe, Pipe, DevNull) +func TestEcho(t *testing.T) { + bs, err := helperCommand("echo", "foo bar", "baz").Output() if err != nil { - t.Fatal("run:", err) - } - io.WriteString(cmd.Stdin, "hello, world\n") - cmd.Stdin.Close() - buf, err := ioutil.ReadAll(cmd.Stdout) - if err != nil { - t.Fatal("read:", err) - } - if string(buf) != "hello, world\n" { - t.Fatalf("read: got %q", buf) + t.Errorf("echo: %v", err) } - if err = cmd.Close(); err != nil { - t.Fatal("close:", err) + if g, e := string(bs), "foo bar baz\n"; g != e { + t.Errorf("echo: want %q, got %q", e, g) } } -func TestRunEcho(t *testing.T) { - cmd, err := run([]string{"sh", "-c", "echo hello world"}, - DevNull, Pipe, DevNull) +func TestCatStdin(t *testing.T) { + // Cat, testing stdin and stdout. + input := "Input string\nLine 2" + p := helperCommand("cat") + p.Stdin = strings.NewReader(input) + bs, err := p.Output() if err != nil { - t.Fatal("run:", err) + t.Errorf("cat: %v", err) } - buf, err := ioutil.ReadAll(cmd.Stdout) - if err != nil { - t.Fatal("read:", err) - } - if string(buf) != "hello world\n" { - t.Fatalf("read: got %q", buf) - } - if err = cmd.Close(); err != nil { - t.Fatal("close:", err) + s := string(bs) + if s != input { + t.Errorf("cat: want %q, got %q", input, s) } } -func TestStderr(t *testing.T) { - cmd, err := run([]string{"sh", "-c", "echo hello world 1>&2"}, - DevNull, DevNull, Pipe) - if err != nil { - t.Fatal("run:", err) +func TestCatGoodAndBadFile(t *testing.T) { + // Testing combined output and error values. + bs, err := helperCommand("cat", "/bogus/file.foo", "exec_test.go").CombinedOutput() + if _, ok := err.(*os.Waitmsg); !ok { + t.Errorf("expected Waitmsg from cat combined; got %T: %v", err, err) } - buf, err := ioutil.ReadAll(cmd.Stderr) - if err != nil { - t.Fatal("read:", err) + s := string(bs) + sp := strings.Split(s, "\n", 2) + if len(sp) != 2 { + t.Fatalf("expected two lines from cat; got %q", s) } - if string(buf) != "hello world\n" { - t.Fatalf("read: got %q", buf) + errLine, body := sp[0], sp[1] + if !strings.HasPrefix(errLine, "Error: open /bogus/file.foo") { + t.Errorf("expected stderr to complain about file; got %q", errLine) } - if err = cmd.Close(); err != nil { - t.Fatal("close:", err) + if !strings.Contains(body, "func TestHelperProcess(t *testing.T)") { + t.Errorf("expected test code; got %q (len %d)", body, len(body)) } } -func TestMergeWithStdout(t *testing.T) { - cmd, err := run([]string{"sh", "-c", "echo hello world 1>&2"}, - DevNull, Pipe, MergeWithStdout) - if err != nil { - t.Fatal("run:", err) - } - buf, err := ioutil.ReadAll(cmd.Stdout) - if err != nil { - t.Fatal("read:", err) - } - if string(buf) != "hello world\n" { - t.Fatalf("read: got %q", buf) + +func TestNoExistBinary(t *testing.T) { + // Can't run a non-existent binary + err := Command("/no-exist-binary").Run() + if err == nil { + t.Error("expected error from /no-exist-binary") } - if err = cmd.Close(); err != nil { - t.Fatal("close:", err) +} + +func TestExitStatus(t *testing.T) { + // Test that exit values are returned correctly + err := helperCommand("exit", "42").Run() + if werr, ok := err.(*os.Waitmsg); ok { + if s, e := werr.String(), "exit status 42"; s != e { + t.Errorf("from exit 42 got exit %q, want %q", s, e) + } + } else { + t.Fatalf("expected Waitmsg from exit 42; got %T: %v", err, err) } } -func TestAddEnvVar(t *testing.T) { - err := os.Setenv("NEWVAR", "hello world") - if err != nil { - t.Fatal("setenv:", err) +func TestPipes(t *testing.T) { + check := func(what string, err os.Error) { + if err != nil { + t.Fatalf("%s: %v", what, err) + } } - cmd, err := run([]string{"sh", "-c", "echo $NEWVAR"}, - DevNull, Pipe, DevNull) - if err != nil { - t.Fatal("run:", err) + // Cat, testing stdin and stdout. + c := helperCommand("pipetest") + stdin, err := c.StdinPipe() + check("StdinPipe", err) + stdout, err := c.StdoutPipe() + check("StdoutPipe", err) + stderr, err := c.StderrPipe() + check("StderrPipe", err) + + outbr := bufio.NewReader(stdout) + errbr := bufio.NewReader(stderr) + line := func(what string, br *bufio.Reader) string { + line, _, err := br.ReadLine() + if err != nil { + t.Fatalf("%s: %v", what, err) + } + return string(line) } - buf, err := ioutil.ReadAll(cmd.Stdout) - if err != nil { - t.Fatal("read:", err) + + err = c.Start() + check("Start", err) + + _, err = stdin.Write([]byte("O:I am output\n")) + check("first stdin Write", err) + if g, e := line("first output line", outbr), "O:I am output"; g != e { + t.Errorf("got %q, want %q", g, e) } - if string(buf) != "hello world\n" { - t.Fatalf("read: got %q", buf) + + _, err = stdin.Write([]byte("E:I am error\n")) + check("second stdin Write", err) + if g, e := line("first error line", errbr), "E:I am error"; g != e { + t.Errorf("got %q, want %q", g, e) } - if err = cmd.Close(); err != nil { - t.Fatal("close:", err) + + _, err = stdin.Write([]byte("O:I am output2\n")) + check("third stdin Write 3", err) + if g, e := line("second output line", outbr), "O:I am output2"; g != e { + t.Errorf("got %q, want %q", g, e) } -} -var tryargs = []string{ - `2`, - `2 `, - "2 \t", - `2" "`, - `2 ab `, - `2 "ab" `, - `2 \ `, - `2 \\ `, - `2 \" `, - `2 \`, - `2\`, - `2"`, - `2\"`, - `2 "`, - `2 \"`, - ``, - `2 ^ `, - `2 \^`, + stdin.Close() + err = c.Wait() + check("Wait", err) } -func TestArgs(t *testing.T) { - for _, a := range tryargs { - argv := []string{ - "awk", - `BEGIN{printf("%s|%s|%s",ARGV[1],ARGV[2],ARGV[3])}`, - "/dev/null", - a, - "EOF", - } - exe, err := LookPath(argv[0]) - if err != nil { - t.Fatal("run:", err) +// TestHelperProcess isn't a real test. It's used as a helper process +// for TestParameterRun. +func TestHelperProcess(*testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + defer os.Exit(0) + + args := os.Args + for len(args) > 0 { + if args[0] == "--" { + args = args[1:] + break } - cmd, err := Run(exe, argv, nil, "", DevNull, Pipe, DevNull) - if err != nil { - t.Fatal("run:", err) + args = args[1:] + } + if len(args) == 0 { + fmt.Fprintf(os.Stderr, "No command\n") + os.Exit(2) + } + + cmd, args := args[0], args[1:] + switch cmd { + case "echo": + iargs := []interface{}{} + for _, s := range args { + iargs = append(iargs, s) } - buf, err := ioutil.ReadAll(cmd.Stdout) - if err != nil { - t.Fatal("read:", err) + fmt.Println(iargs...) + case "cat": + if len(args) == 0 { + io.Copy(os.Stdout, os.Stdin) + return } - expect := "/dev/null|" + a + "|EOF" - if string(buf) != expect { - t.Errorf("read: got %q expect %q", buf, expect) + exit := 0 + for _, fn := range args { + f, err := os.Open(fn) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + exit = 2 + } else { + defer f.Close() + io.Copy(os.Stdout, f) + } } - if err = cmd.Close(); err != nil { - t.Fatal("close:", err) + os.Exit(exit) + case "pipetest": + bufr := bufio.NewReader(os.Stdin) + for { + line, _, err := bufr.ReadLine() + if err == os.EOF { + break + } else if err != nil { + os.Exit(1) + } + if bytes.HasPrefix(line, []byte("O:")) { + os.Stdout.Write(line) + os.Stdout.Write([]byte{'\n'}) + } else if bytes.HasPrefix(line, []byte("E:")) { + os.Stderr.Write(line) + os.Stderr.Write([]byte{'\n'}) + } else { + os.Exit(1) + } } + case "exit": + n, _ := strconv.Atoi(args[0]) + os.Exit(n) + default: + fmt.Fprintf(os.Stderr, "Unknown command %q\n", cmd) + os.Exit(2) } } diff --git a/src/pkg/exec/lp_test.go b/src/pkg/exec/lp_test.go index 54081771e..77d8e848c 100644 --- a/src/pkg/exec/lp_test.go +++ b/src/pkg/exec/lp_test.go @@ -22,12 +22,12 @@ func TestLookPathNotFound(t *testing.T) { if path != "" { t.Fatalf("LookPath path == %q when err != nil", path) } - perr, ok := err.(*PathError) + perr, ok := err.(*Error) if !ok { - t.Fatal("LookPath error is not a PathError") + t.Fatal("LookPath error is not an exec.Error") } if perr.Name != name { - t.Fatalf("want PathError name %q, got %q", name, perr.Name) + t.Fatalf("want Error name %q, got %q", name, perr.Name) } } } diff --git a/src/pkg/exec/lp_unix.go b/src/pkg/exec/lp_unix.go index 44f84347b..3fc3be832 100644 --- a/src/pkg/exec/lp_unix.go +++ b/src/pkg/exec/lp_unix.go @@ -9,12 +9,18 @@ import ( "strings" ) -func canExec(file string) bool { +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = os.ErrorString("executable file not found in $PATH") + +func findExecutable(file string) os.Error { d, err := os.Stat(file) if err != nil { - return false + return err + } + if d.IsRegular() && d.Permission()&0111 != 0 { + return nil } - return d.IsRegular() && d.Permission()&0111 != 0 + return os.EPERM } // LookPath searches for an executable binary named file @@ -26,10 +32,11 @@ func LookPath(file string) (string, os.Error) { // but that would not match all the Unix shells. if strings.Contains(file, "/") { - if canExec(file) { + err := findExecutable(file) + if err == nil { return file, nil } - return "", &PathError{file} + return "", &Error{file, err} } pathenv := os.Getenv("PATH") for _, dir := range strings.Split(pathenv, ":", -1) { @@ -37,9 +44,9 @@ func LookPath(file string) (string, os.Error) { // Unix shell semantics: path element "" means "." dir = "." } - if canExec(dir + "/" + file) { + if err := findExecutable(dir + "/" + file); err == nil { return dir + "/" + file, nil } } - return "", &PathError{file} + return "", &Error{file, ErrNotFound} } diff --git a/src/pkg/exec/lp_windows.go b/src/pkg/exec/lp_windows.go index d357575fd..758861021 100644 --- a/src/pkg/exec/lp_windows.go +++ b/src/pkg/exec/lp_windows.go @@ -9,15 +9,21 @@ import ( "strings" ) -func chkStat(file string) bool { +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = os.ErrorString("executable file not found in %PATH%") + +func chkStat(file string) os.Error { d, err := os.Stat(file) if err != nil { - return false + return err + } + if d.IsRegular() { + return nil } - return d.IsRegular() + return os.EPERM } -func canExec(file string, exts []string) (string, bool) { +func findExecutable(file string, exts []string) (string, os.Error) { if len(exts) == 0 { return file, chkStat(file) } @@ -28,14 +34,14 @@ func canExec(file string, exts []string) (string, bool) { } } for _, e := range exts { - if f := file + e; chkStat(f) { - return f, true + if f := file + e; chkStat(f) == nil { + return f, nil } } - return ``, false + return ``, ErrNotFound } -func LookPath(file string) (string, os.Error) { +func LookPath(file string) (f string, err os.Error) { exts := []string{} if x := os.Getenv(`PATHEXT`); x != `` { exts = strings.Split(strings.ToLower(x), `;`, -1) @@ -46,21 +52,21 @@ func LookPath(file string) (string, os.Error) { } } if strings.Contains(file, `\`) || strings.Contains(file, `/`) { - if f, ok := canExec(file, exts); ok { - return f, nil + if f, err = findExecutable(file, exts); err == nil { + return } - return ``, &PathError{file} + return ``, &Error{file, err} } if pathenv := os.Getenv(`PATH`); pathenv == `` { - if f, ok := canExec(`.\`+file, exts); ok { - return f, nil + if f, err = findExecutable(`.\`+file, exts); err == nil { + return } } else { for _, dir := range strings.Split(pathenv, `;`, -1) { - if f, ok := canExec(dir+`\`+file, exts); ok { - return f, nil + if f, err = findExecutable(dir+`\`+file, exts); err == nil { + return } } } - return ``, &PathError{file} + return ``, &Error{file, ErrNotFound} } diff --git a/src/pkg/exp/datafmt/datafmt.go b/src/pkg/exp/datafmt/datafmt.go index a8efdc58f..10e4b54f9 100644 --- a/src/pkg/exp/datafmt/datafmt.go +++ b/src/pkg/exp/datafmt/datafmt.go @@ -594,7 +594,7 @@ func (s *State) eval(fexpr expr, value reflect.Value, index int) bool { s.eval(t.indent, value, index) // if the indentation evaluates to nil, the state's output buffer // didn't change - either way it's ok to append the difference to - // the current identation + // the current indentation s.indent.Write(s.output.Bytes()[mark.outputLen:s.output.Len()]) s.restore(mark) diff --git a/src/pkg/exp/eval/expr.go b/src/pkg/exp/eval/expr.go index e65f47617..14a0659b6 100644 --- a/src/pkg/exp/eval/expr.go +++ b/src/pkg/exp/eval/expr.go @@ -1781,7 +1781,7 @@ func (a *exprInfo) compileBinaryExpr(op token.Token, l, r *expr) *expr { // written: Function values are equal if they were // created by the same execution of a function literal // or refer to the same function declaration. This is - // *almost* but not quite waht 6g implements. If a + // *almost* but not quite what 6g implements. If a // function literals does not capture any variables, // then multiple executions of it will result in the // same closure. Russ says he'll change that. diff --git a/src/pkg/exp/eval/gen.go b/src/pkg/exp/eval/gen.go index de98a5d15..1e00bdcd0 100644 --- a/src/pkg/exp/eval/gen.go +++ b/src/pkg/exp/eval/gen.go @@ -366,10 +366,10 @@ func main() { t.SetDelims("«", "»") err := t.Parse(templateStr) if err != nil { - log.Exit(err) + log.Fatal(err) } err = t.Execute(os.Stdout, data) if err != nil { - log.Exit(err) + log.Fatal(err) } } diff --git a/src/pkg/exp/eval/stmt.go b/src/pkg/exp/eval/stmt.go index f6b7c1cda..57bf20e6f 100644 --- a/src/pkg/exp/eval/stmt.go +++ b/src/pkg/exp/eval/stmt.go @@ -68,7 +68,7 @@ type flowBuf struct { gotos map[token.Pos]*flowBlock // labels is a map from label name to information on the block // at the point of the label. labels are tracked by name, - // since mutliple labels at the same PC can have different + // since multiple labels at the same PC can have different // blocks. labels map[string]*flowBlock } @@ -307,7 +307,7 @@ func (a *stmtCompiler) compile(s ast.Stmt) { } if notimpl { - a.diag("%T statment node not implemented", s) + a.diag("%T statement node not implemented", s) } if a.block.inner != nil { @@ -550,7 +550,7 @@ func (a *stmtCompiler) doAssign(lhs []ast.Expr, rhs []ast.Expr, tok token.Token, ident, ok = le.(*ast.Ident) if !ok { a.diagAt(le.Pos(), "left side of := must be a name") - // Suppress new defitions errors + // Suppress new definitions errors nDefs++ continue } diff --git a/src/pkg/exp/eval/typec.go b/src/pkg/exp/eval/typec.go index de90cf664..0ed24a8d2 100644 --- a/src/pkg/exp/eval/typec.go +++ b/src/pkg/exp/eval/typec.go @@ -68,7 +68,7 @@ func (a *typeCompiler) compileArrayType(x *ast.ArrayType, allowRec bool) Type { } if _, ok := x.Len.(*ast.Ellipsis); ok { - a.diagAt(x.Len.Pos(), "... array initailizers not implemented") + a.diagAt(x.Len.Pos(), "... array initializers not implemented") return nil } l, ok := a.compileArrayLen(a.block, x.Len) diff --git a/src/pkg/encoding/line/Makefile b/src/pkg/exp/gui/Makefile index 1af355c27..af065e4a5 100644 --- a/src/pkg/encoding/line/Makefile +++ b/src/pkg/exp/gui/Makefile @@ -1,11 +1,11 @@ -# Copyright 2010 The Go Authors. All rights reserved. +# Copyright 2009 The Go Authors. All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. include ../../../Make.inc -TARG=encoding/line +TARG=exp/gui GOFILES=\ - line.go\ + gui.go\ include ../../../Make.pkg diff --git a/src/pkg/exp/draw/event.go b/src/pkg/exp/gui/gui.go index b777d912e..171499186 100644 --- a/src/pkg/exp/draw/event.go +++ b/src/pkg/exp/gui/gui.go @@ -2,17 +2,19 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package draw +// Package gui defines a basic graphical user interface programming model. +package gui import ( "image" + "image/draw" "os" ) // A Window represents a single graphics window. type Window interface { // Screen returns an editable Image for the window. - Screen() Image + Screen() draw.Image // FlushImage flushes changes made to Screen() back to screen. FlushImage() // EventChan returns a channel carrying UI events such as key presses, diff --git a/src/pkg/exp/draw/x11/Makefile b/src/pkg/exp/gui/x11/Makefile index 205b3a65b..88cc1e23b 100644 --- a/src/pkg/exp/draw/x11/Makefile +++ b/src/pkg/exp/gui/x11/Makefile @@ -4,7 +4,7 @@ include ../../../../Make.inc -TARG=exp/draw/x11 +TARG=exp/gui/x11 GOFILES=\ auth.go\ conn.go\ diff --git a/src/pkg/exp/draw/x11/auth.go b/src/pkg/exp/gui/x11/auth.go index d48936ac1..d48936ac1 100644 --- a/src/pkg/exp/draw/x11/auth.go +++ b/src/pkg/exp/gui/x11/auth.go diff --git a/src/pkg/exp/draw/x11/conn.go b/src/pkg/exp/gui/x11/conn.go index 81c67267d..bc7ca63db 100644 --- a/src/pkg/exp/draw/x11/conn.go +++ b/src/pkg/exp/gui/x11/conn.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package x11 implements an X11 backend for the exp/draw package. +// Package x11 implements an X11 backend for the exp/gui package. // // The X protocol specification is at ftp://ftp.x.org/pub/X11R7.0/doc/PDF/proto.pdf. // A summary of the wire format can be found in XCB's xproto.xml. @@ -10,8 +10,9 @@ package x11 import ( "bufio" - "exp/draw" + "exp/gui" "image" + "image/draw" "io" "log" "net" @@ -43,7 +44,7 @@ type conn struct { img *image.RGBA eventc chan interface{} - mouseState draw.MouseEvent + mouseState gui.MouseEvent buf [256]byte // General purpose scratch buffer. @@ -53,7 +54,7 @@ type conn struct { } // writeSocket runs in its own goroutine, serving both FlushImage calls -// directly from the exp/draw client and indirectly from X expose events. +// directly from the exp/gui client and indirectly from X expose events. // It paints c.img to the X server via PutImage requests. func (c *conn) writeSocket() { defer c.c.Close() @@ -143,7 +144,7 @@ func (c *conn) Close() os.Error { func (c *conn) EventChan() <-chan interface{} { return c.eventc } -// readSocket runs in its own goroutine, reading X events and sending draw +// readSocket runs in its own goroutine, reading X events and sending gui // events on c's EventChan. func (c *conn) readSocket() { var ( @@ -155,7 +156,7 @@ func (c *conn) readSocket() { // X events are always 32 bytes long. if _, err := io.ReadFull(c.r, c.buf[0:32]); err != nil { if err != os.EOF { - c.eventc <- draw.ErrEvent{err} + c.eventc <- gui.ErrEvent{err} } return } @@ -165,7 +166,7 @@ func (c *conn) readSocket() { if cookie != 1 { // We issued only one request (GetKeyboardMapping) with a cookie of 1, // so we shouldn't get any other reply from the X server. - c.eventc <- draw.ErrEvent{os.NewError("x11: unexpected cookie")} + c.eventc <- gui.ErrEvent{os.NewError("x11: unexpected cookie")} return } keysymsPerKeycode = int(c.buf[1]) @@ -179,7 +180,7 @@ func (c *conn) readSocket() { u, err := readU32LE(c.r, c.buf[0:4]) if err != nil { if err != os.EOF { - c.eventc <- draw.ErrEvent{err} + c.eventc <- gui.ErrEvent{err} } return } @@ -204,11 +205,11 @@ func (c *conn) readSocket() { // TODO(nigeltao): Should we send KeyEvents for Shift/Ctrl/Alt? Should Shift-A send // the same int down the channel as the sent on just the A key? // TODO(nigeltao): How should IME events (e.g. key presses that should generate CJK text) work? Or - // is that outside the scope of the draw.Window interface? + // is that outside the scope of the gui.Window interface? if c.buf[0] == 0x03 { keysym = -keysym } - c.eventc <- draw.KeyEvent{keysym} + c.eventc <- gui.KeyEvent{keysym} case 0x04, 0x05: // Button press, button release. mask := 1 << (c.buf[1] - 1) if c.buf[0] == 0x04 { @@ -310,7 +311,7 @@ func authenticate(w *bufio.Writer, displayStr string) os.Error { return os.NewError("unsupported Xauth") } // 0x006c means little-endian. 0x000b, 0x0000 means X major version 11, minor version 0. - // 0x0012 and 0x0010 means the auth key and value have lenths 18 and 16. + // 0x0012 and 0x0010 means the auth key and value have lengths 18 and 16. // The final 0x0000 is padding, so that the string length is a multiple of 4. _, err = io.WriteString(w, "\x6c\x00\x0b\x00\x00\x00\x12\x00\x10\x00\x00\x00") if err != nil { @@ -517,7 +518,7 @@ func (c *conn) handshake() os.Error { if err != nil { return err } - // Ignore some things that we don't care about (totalling 10 + vendorLen bytes): + // Ignore some things that we don't care about (totaling 10 + vendorLen bytes): // imageByteOrder(1), bitmapFormatBitOrder(1), bitmapFormatScanlineUnit(1) bitmapFormatScanlinePad(1), // minKeycode(1), maxKeycode(1), padding(4), vendor (vendorLen). if 10+int(vendorLen) > cap(c.buf) { @@ -551,7 +552,7 @@ func (c *conn) handshake() os.Error { } // NewWindow calls NewWindowDisplay with $DISPLAY. -func NewWindow() (draw.Window, os.Error) { +func NewWindow() (gui.Window, os.Error) { display := os.Getenv("DISPLAY") if len(display) == 0 { return nil, os.NewError("$DISPLAY not set") @@ -559,10 +560,10 @@ func NewWindow() (draw.Window, os.Error) { return NewWindowDisplay(display) } -// NewWindowDisplay returns a new draw.Window, backed by a newly created and +// NewWindowDisplay returns a new gui.Window, backed by a newly created and // mapped X11 window. The X server to connect to is specified by the display // string, such as ":1". -func NewWindowDisplay(display string) (draw.Window, os.Error) { +func NewWindowDisplay(display string) (gui.Window, os.Error) { socket, displayStr, err := connect(display) if err != nil { return nil, err diff --git a/src/pkg/exp/wingui/Makefile b/src/pkg/exp/wingui/Makefile index 983a8270b..e382c019f 100644 --- a/src/pkg/exp/wingui/Makefile +++ b/src/pkg/exp/wingui/Makefile @@ -18,7 +18,7 @@ GOFILES=\ include ../../../Make.cmd zwinapi.go: winapi.go - $(GOROOT)/src/pkg/syscall/mksyscall_windows.sh $< \ + $(GOROOT)/src/pkg/syscall/mksyscall_windows.pl $< \ | sed 's/^package.*syscall$$/package main/' \ | sed '/^import/a \ import "syscall"' \ diff --git a/src/pkg/exp/wingui/winapi.go b/src/pkg/exp/wingui/winapi.go index c96f45299..fb0d61009 100644 --- a/src/pkg/exp/wingui/winapi.go +++ b/src/pkg/exp/wingui/winapi.go @@ -96,7 +96,7 @@ const ( // Some button control styles BS_DEFPUSHBUTTON = 1 - // Some colour constants + // Some color constants COLOR_WINDOW = 5 COLOR_BTNFACE = 15 @@ -108,13 +108,13 @@ const ( ) var ( - // Some globaly known cusrors + // Some globally known cursors IDC_ARROW = MakeIntResource(32512) IDC_IBEAM = MakeIntResource(32513) IDC_WAIT = MakeIntResource(32514) IDC_CROSS = MakeIntResource(32515) - // Some globaly known icons + // Some globally known icons IDI_APPLICATION = MakeIntResource(32512) IDI_HAND = MakeIntResource(32513) IDI_QUESTION = MakeIntResource(32514) diff --git a/src/pkg/exp/wingui/zwinapi.go b/src/pkg/exp/wingui/zwinapi.go index 60aaac6cf..6ae6330a1 100644 --- a/src/pkg/exp/wingui/zwinapi.go +++ b/src/pkg/exp/wingui/zwinapi.go @@ -1,4 +1,4 @@ -// mksyscall_windows.sh winapi.go +// mksyscall_windows.pl winapi.go // MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT package main diff --git a/src/pkg/expvar/expvar_test.go b/src/pkg/expvar/expvar_test.go index 94926d9f8..8f7a48168 100644 --- a/src/pkg/expvar/expvar_test.go +++ b/src/pkg/expvar/expvar_test.go @@ -76,33 +76,33 @@ func TestString(t *testing.T) { } func TestMapCounter(t *testing.T) { - colours := NewMap("bike-shed-colours") + colors := NewMap("bike-shed-colors") - colours.Add("red", 1) - colours.Add("red", 2) - colours.Add("blue", 4) - colours.AddFloat("green", 4.125) - if x := colours.m["red"].(*Int).i; x != 3 { - t.Errorf("colours.m[\"red\"] = %v, want 3", x) + colors.Add("red", 1) + colors.Add("red", 2) + colors.Add("blue", 4) + colors.AddFloat("green", 4.125) + if x := colors.m["red"].(*Int).i; x != 3 { + t.Errorf("colors.m[\"red\"] = %v, want 3", x) } - if x := colours.m["blue"].(*Int).i; x != 4 { - t.Errorf("colours.m[\"blue\"] = %v, want 4", x) + if x := colors.m["blue"].(*Int).i; x != 4 { + t.Errorf("colors.m[\"blue\"] = %v, want 4", x) } - if x := colours.m["green"].(*Float).f; x != 4.125 { - t.Errorf("colours.m[\"green\"] = %v, want 3.14", x) + if x := colors.m["green"].(*Float).f; x != 4.125 { + t.Errorf("colors.m[\"green\"] = %v, want 3.14", x) } - // colours.String() should be '{"red":3, "blue":4}', + // colors.String() should be '{"red":3, "blue":4}', // though the order of red and blue could vary. - s := colours.String() + s := colors.String() var j interface{} err := json.Unmarshal([]byte(s), &j) if err != nil { - t.Errorf("colours.String() isn't valid JSON: %v", err) + t.Errorf("colors.String() isn't valid JSON: %v", err) } m, ok := j.(map[string]interface{}) if !ok { - t.Error("colours.String() didn't produce a map.") + t.Error("colors.String() didn't produce a map.") } red := m["red"] x, ok := red.(float64) diff --git a/src/pkg/flag/export_test.go b/src/pkg/flag/export_test.go index b5e3243b3..7b190807a 100644 --- a/src/pkg/flag/export_test.go +++ b/src/pkg/flag/export_test.go @@ -9,24 +9,14 @@ import "os" // Additional routines compiled into the package only during testing. // ResetForTesting clears all flag state and sets the usage function as directed. -// After calling ResetForTesting, parse errors in flag handling will panic rather -// than exit the program. +// After calling ResetForTesting, parse errors in flag handling will not +// exit the program. func ResetForTesting(usage func()) { - flags = &allFlags{make(map[string]*Flag), make(map[string]*Flag), os.Args[1:]} + commandLine = NewFlagSet(os.Args[0], ContinueOnError) Usage = usage - panicOnError = true } -// ParseForTesting parses the flag state using the provided arguments. It -// should be called after 1) ResetForTesting and 2) setting up the new flags. -// The return value reports whether the parse was error-free. -func ParseForTesting(args []string) (result bool) { - defer func() { - if recover() != nil { - result = false - } - }() - os.Args = args - Parse() - return true +// CommandLine returns the default FlagSet. +func CommandLine() *FlagSet { + return commandLine } diff --git a/src/pkg/flag/flag.go b/src/pkg/flag/flag.go index 9ed20e06b..e5d2f94e9 100644 --- a/src/pkg/flag/flag.go +++ b/src/pkg/flag/flag.go @@ -50,18 +50,12 @@ Integer flags accept 1234, 0664, 0x1234 and may be negative. Boolean flags may be 1, 0, t, f, true, false, TRUE, FALSE, True, False. - It is safe to call flag.Parse multiple times, possibly after changing - os.Args. This makes it possible to implement command lines with - subcommands that enable additional flags, as in: - - flag.Bool(...) // global options - flag.Parse() // parse leading command - subcmd := flag.Arg(0) - switch subcmd { - // add per-subcommand options - } - os.Args = flag.Args() - flag.Parse() + The default set of command-line flags is controlled by + top-level functions. The FlagSet type allows one to define + independent sets of flags, such as to implement subcommands + in a command-line interface. The methods of FlagSet are + analogous to the top-level functions for the command-line + flag set. */ package flag @@ -190,6 +184,30 @@ type Value interface { Set(string) bool } +// ErrorHandling defines how to handle flag parsing errors. +type ErrorHandling int + +const ( + ContinueOnError ErrorHandling = iota + ExitOnError + PanicOnError +) + +// A FlagSet represents a set of defined flags. +type FlagSet struct { + // Usage is the function called when an error occurs while parsing flags. + // The field is a function (not a method) that may be changed to point to + // a custom error handler. + Usage func() + + name string + actual map[string]*Flag + formal map[string]*Flag + args []string // arguments after flags + exitOnError bool // does the program exit if there's an error? + errorHandling ErrorHandling +} + // A Flag represents the state of a flag. type Flag struct { Name string // name as it appears on command line @@ -198,14 +216,6 @@ type Flag struct { DefValue string // default value (as text); for usage message } -type allFlags struct { - actual map[string]*Flag - formal map[string]*Flag - args []string // arguments after flags -} - -var flags *allFlags - // sortFlags returns the flags as a slice in lexicographical sorted order. func sortFlags(flags map[string]*Flag) []*Flag { list := make(sort.StringArray, len(flags)) @@ -224,43 +234,67 @@ func sortFlags(flags map[string]*Flag) []*Flag { // VisitAll visits the flags in lexicographical order, calling fn for each. // It visits all flags, even those not set. -func VisitAll(fn func(*Flag)) { - for _, f := range sortFlags(flags.formal) { - fn(f) +func (f *FlagSet) VisitAll(fn func(*Flag)) { + for _, flag := range sortFlags(f.formal) { + fn(flag) } } +// VisitAll visits the command-line flags in lexicographical order, calling +// fn for each. It visits all flags, even those not set. +func VisitAll(fn func(*Flag)) { + commandLine.VisitAll(fn) +} + // Visit visits the flags in lexicographical order, calling fn for each. // It visits only those flags that have been set. -func Visit(fn func(*Flag)) { - for _, f := range sortFlags(flags.actual) { - fn(f) +func (f *FlagSet) Visit(fn func(*Flag)) { + for _, flag := range sortFlags(f.actual) { + fn(flag) } } +// Visit visits the command-line flags in lexicographical order, calling fn +// for each. It visits only those flags that have been set. +func Visit(fn func(*Flag)) { + commandLine.Visit(fn) +} + // Lookup returns the Flag structure of the named flag, returning nil if none exists. +func (f *FlagSet) Lookup(name string) *Flag { + return f.formal[name] +} + +// Lookup returns the Flag structure of the named command-line flag, +// returning nil if none exists. func Lookup(name string) *Flag { - return flags.formal[name] + return commandLine.formal[name] } // Set sets the value of the named flag. It returns true if the set succeeded; false if // there is no such flag defined. -func Set(name, value string) bool { - f, ok := flags.formal[name] +func (f *FlagSet) Set(name, value string) bool { + flag, ok := f.formal[name] if !ok { return false } - ok = f.Value.Set(value) + ok = flag.Value.Set(value) if !ok { return false } - flags.actual[name] = f + f.actual[name] = flag return true } -// PrintDefaults prints to standard error the default values of all defined flags. -func PrintDefaults() { - VisitAll(func(f *Flag) { +// Set sets the value of the named command-line flag. It returns true if the +// set succeeded; false if there is no such flag defined. +func Set(name, value string) bool { + return commandLine.Set(name, value) +} + +// PrintDefaults prints to standard error the default values of all defined flags in the set. +func (f *FlagSet) PrintDefaults() { + f.VisitAll(func(f *Flag) { format := " -%s=%s: %s\n" if _, ok := f.Value.(*stringValue); ok { // put quotes on the value @@ -270,174 +304,298 @@ func PrintDefaults() { }) } -// Usage prints to standard error a default usage message documenting all defined flags. +// PrintDefaults prints to standard error the default values of all defined command-line flags. +func PrintDefaults() { + commandLine.PrintDefaults() +} + +// defaultUsage is the default function to print a usage message. +func defaultUsage(f *FlagSet) { + fmt.Fprintf(os.Stderr, "Usage of %s:\n", f.name) + f.PrintDefaults() +} + +// Usage prints to standard error a usage message documenting all defined command-line flags. // The function is a variable that may be changed to point to a custom function. var Usage = func() { - fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) - PrintDefaults() + defaultUsage(commandLine) } -var panicOnError = false +// NFlag returns the number of flags that have been set. +func (f *FlagSet) NFlag() int { return len(f.actual) } -// failf prints to standard error a formatted error and Usage, and then exits the program. -func failf(format string, a ...interface{}) { - fmt.Fprintf(os.Stderr, format, a...) - Usage() - if panicOnError { - panic("flag parse error") +// NFlag returns the number of command-line flags that have been set. +func NFlag() int { return len(commandLine.actual) } + +// Arg returns the i'th argument. Arg(0) is the first remaining argument +// after flags have been processed. +func (f *FlagSet) Arg(i int) string { + if i < 0 || i >= len(f.args) { + return "" } - os.Exit(2) + return f.args[i] } -// NFlag returns the number of flags that have been set. -func NFlag() int { return len(flags.actual) } - // Arg returns the i'th command-line argument. Arg(0) is the first remaining argument // after flags have been processed. func Arg(i int) string { - if i < 0 || i >= len(flags.args) { - return "" - } - return flags.args[i] + return commandLine.Arg(i) } // NArg is the number of arguments remaining after flags have been processed. -func NArg() int { return len(flags.args) } +func (f *FlagSet) NArg() int { return len(f.args) } + +// NArg is the number of arguments remaining after flags have been processed. +func NArg() int { return len(commandLine.args) } + +// Args returns the non-flag arguments. +func (f *FlagSet) Args() []string { return f.args } // Args returns the non-flag command-line arguments. -func Args() []string { return flags.args } +func Args() []string { return commandLine.args } + +// BoolVar defines a bool flag with specified name, default value, and usage string. +// The argument p points to a bool variable in which to store the value of the flag. +func (f *FlagSet) BoolVar(p *bool, name string, value bool, usage string) { + f.Var(newBoolValue(value, p), name, usage) +} // BoolVar defines a bool flag with specified name, default value, and usage string. // The argument p points to a bool variable in which to store the value of the flag. func BoolVar(p *bool, name string, value bool, usage string) { - Var(newBoolValue(value, p), name, usage) + commandLine.Var(newBoolValue(value, p), name, usage) } // Bool defines a bool flag with specified name, default value, and usage string. // The return value is the address of a bool variable that stores the value of the flag. -func Bool(name string, value bool, usage string) *bool { +func (f *FlagSet) Bool(name string, value bool, usage string) *bool { p := new(bool) - BoolVar(p, name, value, usage) + f.BoolVar(p, name, value, usage) return p } +// Bool defines a bool flag with specified name, default value, and usage string. +// The return value is the address of a bool variable that stores the value of the flag. +func Bool(name string, value bool, usage string) *bool { + return commandLine.Bool(name, value, usage) +} + +// IntVar defines an int flag with specified name, default value, and usage string. +// The argument p points to an int variable in which to store the value of the flag. +func (f *FlagSet) IntVar(p *int, name string, value int, usage string) { + f.Var(newIntValue(value, p), name, usage) +} + // IntVar defines an int flag with specified name, default value, and usage string. // The argument p points to an int variable in which to store the value of the flag. func IntVar(p *int, name string, value int, usage string) { - Var(newIntValue(value, p), name, usage) + commandLine.Var(newIntValue(value, p), name, usage) } // Int defines an int flag with specified name, default value, and usage string. // The return value is the address of an int variable that stores the value of the flag. -func Int(name string, value int, usage string) *int { +func (f *FlagSet) Int(name string, value int, usage string) *int { p := new(int) - IntVar(p, name, value, usage) + f.IntVar(p, name, value, usage) return p } +// Int defines an int flag with specified name, default value, and usage string. +// The return value is the address of an int variable that stores the value of the flag. +func Int(name string, value int, usage string) *int { + return commandLine.Int(name, value, usage) +} + +// Int64Var defines an int64 flag with specified name, default value, and usage string. +// The argument p points to an int64 variable in which to store the value of the flag. +func (f *FlagSet) Int64Var(p *int64, name string, value int64, usage string) { + f.Var(newInt64Value(value, p), name, usage) +} + // Int64Var defines an int64 flag with specified name, default value, and usage string. // The argument p points to an int64 variable in which to store the value of the flag. func Int64Var(p *int64, name string, value int64, usage string) { - Var(newInt64Value(value, p), name, usage) + commandLine.Var(newInt64Value(value, p), name, usage) } // Int64 defines an int64 flag with specified name, default value, and usage string. // The return value is the address of an int64 variable that stores the value of the flag. -func Int64(name string, value int64, usage string) *int64 { +func (f *FlagSet) Int64(name string, value int64, usage string) *int64 { p := new(int64) - Int64Var(p, name, value, usage) + f.Int64Var(p, name, value, usage) return p } +// Int64 defines an int64 flag with specified name, default value, and usage string. +// The return value is the address of an int64 variable that stores the value of the flag. +func Int64(name string, value int64, usage string) *int64 { + return commandLine.Int64(name, value, usage) +} + // UintVar defines a uint flag with specified name, default value, and usage string. // The argument p points to a uint variable in which to store the value of the flag. +func (f *FlagSet) UintVar(p *uint, name string, value uint, usage string) { + f.Var(newUintValue(value, p), name, usage) +} + +// UintVar defines a uint flag with specified name, default value, and usage string. +// The argument p points to a uint variable in which to store the value of the flag. func UintVar(p *uint, name string, value uint, usage string) { - Var(newUintValue(value, p), name, usage) + commandLine.Var(newUintValue(value, p), name, usage) } // Uint defines a uint flag with specified name, default value, and usage string. -// The return value is the address of a uint variable that stores the value of the flag. -func Uint(name string, value uint, usage string) *uint { +// The return value is the address of a uint variable that stores the value of the flag. +func (f *FlagSet) Uint(name string, value uint, usage string) *uint { p := new(uint) - UintVar(p, name, value, usage) + f.UintVar(p, name, value, usage) return p } +// Uint defines a uint flag with specified name, default value, and usage string. +// The return value is the address of a uint variable that stores the value of the flag. +func Uint(name string, value uint, usage string) *uint { + return commandLine.Uint(name, value, usage) +} + +// Uint64Var defines a uint64 flag with specified name, default value, and usage string. +// The argument p points to a uint64 variable in which to store the value of the flag. +func (f *FlagSet) Uint64Var(p *uint64, name string, value uint64, usage string) { + f.Var(newUint64Value(value, p), name, usage) +} + // Uint64Var defines a uint64 flag with specified name, default value, and usage string. // The argument p points to a uint64 variable in which to store the value of the flag. func Uint64Var(p *uint64, name string, value uint64, usage string) { - Var(newUint64Value(value, p), name, usage) + commandLine.Var(newUint64Value(value, p), name, usage) } // Uint64 defines a uint64 flag with specified name, default value, and usage string. // The return value is the address of a uint64 variable that stores the value of the flag. -func Uint64(name string, value uint64, usage string) *uint64 { +func (f *FlagSet) Uint64(name string, value uint64, usage string) *uint64 { p := new(uint64) - Uint64Var(p, name, value, usage) + f.Uint64Var(p, name, value, usage) return p } +// Uint64 defines a uint64 flag with specified name, default value, and usage string. +// The return value is the address of a uint64 variable that stores the value of the flag. +func Uint64(name string, value uint64, usage string) *uint64 { + return commandLine.Uint64(name, value, usage) +} + +// StringVar defines a string flag with specified name, default value, and usage string. +// The argument p points to a string variable in which to store the value of the flag. +func (f *FlagSet) StringVar(p *string, name string, value string, usage string) { + f.Var(newStringValue(value, p), name, usage) +} + // StringVar defines a string flag with specified name, default value, and usage string. // The argument p points to a string variable in which to store the value of the flag. -func StringVar(p *string, name, value string, usage string) { - Var(newStringValue(value, p), name, usage) +func StringVar(p *string, name string, value string, usage string) { + commandLine.Var(newStringValue(value, p), name, usage) } // String defines a string flag with specified name, default value, and usage string. // The return value is the address of a string variable that stores the value of the flag. -func String(name, value string, usage string) *string { +func (f *FlagSet) String(name string, value string, usage string) *string { p := new(string) - StringVar(p, name, value, usage) + f.StringVar(p, name, value, usage) return p } +// String defines a string flag with specified name, default value, and usage string. +// The return value is the address of a string variable that stores the value of the flag. +func String(name string, value string, usage string) *string { + return commandLine.String(name, value, usage) +} + +// Float64Var defines a float64 flag with specified name, default value, and usage string. +// The argument p points to a float64 variable in which to store the value of the flag. +func (f *FlagSet) Float64Var(p *float64, name string, value float64, usage string) { + f.Var(newFloat64Value(value, p), name, usage) +} + // Float64Var defines a float64 flag with specified name, default value, and usage string. // The argument p points to a float64 variable in which to store the value of the flag. func Float64Var(p *float64, name string, value float64, usage string) { - Var(newFloat64Value(value, p), name, usage) + commandLine.Var(newFloat64Value(value, p), name, usage) } // Float64 defines a float64 flag with specified name, default value, and usage string. // The return value is the address of a float64 variable that stores the value of the flag. -func Float64(name string, value float64, usage string) *float64 { +func (f *FlagSet) Float64(name string, value float64, usage string) *float64 { p := new(float64) - Float64Var(p, name, value, usage) + f.Float64Var(p, name, value, usage) return p } -// Var defines a user-typed flag with specified name, default value, and usage string. -// The argument p points to a Value variable in which to store the value of the flag. -func Var(value Value, name string, usage string) { +// Float64 defines an int flag with specified name, default value, and usage string. +// The return value is the address of a float64 variable that stores the value of the flag. +func Float64(name string, value float64, usage string) *float64 { + return commandLine.Float64(name, value, usage) +} + +// Var defines a flag with the specified name and usage string. The type and +// value of the flag are represented by the first argument, of type Value, which +// typically holds a user-defined implementation of Value. For instance, the +// caller could create a flag that turns a comma-separated string into a slice +// of strings by giving the slice the methods of Value; in particular, Set would +// decompose the comma-separated string into the slice. +func (f *FlagSet) Var(value Value, name string, usage string) { // Remember the default value as a string; it won't change. - f := &Flag{name, usage, value, value.String()} - _, alreadythere := flags.formal[name] + flag := &Flag{name, usage, value, value.String()} + _, alreadythere := f.formal[name] if alreadythere { - fmt.Fprintln(os.Stderr, "flag redefined:", name) + fmt.Fprintf(os.Stderr, "%s flag redefined: %s\n", f.name, name) panic("flag redefinition") // Happens only if flags are declared with identical names } - flags.formal[name] = f + f.formal[name] = flag } +// Var defines a flag with the specified name and usage string. The type and +// value of the flag are represented by the first argument, of type Value, which +// typically holds a user-defined implementation of Value. For instance, the +// caller could create a flag that turns a comma-separated string into a slice +// of strings by giving the slice the methods of Value; in particular, Set would +// decompose the comma-separated string into the slice. +func Var(value Value, name string, usage string) { + commandLine.Var(value, name, usage) +} -func (f *allFlags) parseOne() (ok bool) { +// failf prints to standard error a formatted error and usage message and +// returns the error. +func (f *FlagSet) failf(format string, a ...interface{}) os.Error { + err := fmt.Errorf(format, a...) + fmt.Fprintln(os.Stderr, err) + if f == commandLine { + Usage() + } else { + f.Usage() + } + return err +} + +// parseOne parses one flag. It returns whether a flag was seen. +func (f *FlagSet) parseOne() (bool, os.Error) { if len(f.args) == 0 { - return false + return false, nil } s := f.args[0] if len(s) == 0 || s[0] != '-' || len(s) == 1 { - return false + return false, nil } num_minuses := 1 if s[1] == '-' { num_minuses++ if len(s) == 2 { // "--" terminates the flags f.args = f.args[1:] - return false + return false, nil } } name := s[num_minuses:] if len(name) == 0 || name[0] == '-' || name[0] == '=' { - failf("bad flag syntax: %s\n", s) + return false, f.failf("bad flag syntax: %s", s) } // it's a flag. does it have an argument? @@ -452,15 +610,15 @@ func (f *allFlags) parseOne() (ok bool) { break } } - m := flags.formal + m := f.formal flag, alreadythere := m[name] // BUG if !alreadythere { - failf("flag provided but not defined: -%s\n", name) + return false, f.failf("flag provided but not defined: -%s", name) } if fv, ok := flag.Value.(*boolValue); ok { // special case: doesn't need an arg if has_value { if !fv.Set(value) { - failf("invalid boolean value %q for flag: -%s\n", value, name) + f.failf("invalid boolean value %q for flag: -%s", value, name) } } else { fv.Set("true") @@ -473,25 +631,61 @@ func (f *allFlags) parseOne() (ok bool) { value, f.args = f.args[0], f.args[1:] } if !has_value { - failf("flag needs an argument: -%s\n", name) + return false, f.failf("flag needs an argument: -%s", name) } ok = flag.Value.Set(value) if !ok { - failf("invalid value %q for flag: -%s\n", value, name) + return false, f.failf("invalid value %q for flag: -%s", value, name) } } - flags.actual[name] = flag - return true + f.actual[name] = flag + return true, nil +} + +// Parse parses flag definitions from the argument list, which should not +// include the command name. Must be called after all flags in the FlagSet +// are defined and before flags are accessed by the program. +func (f *FlagSet) Parse(arguments []string) os.Error { + f.args = arguments + for { + seen, err := f.parseOne() + if seen { + continue + } + if err == nil { + break + } + switch f.errorHandling { + case ContinueOnError: + return err + case ExitOnError: + os.Exit(2) + case PanicOnError: + panic(err) + } + } + return nil } -// Parse parses the command-line flags. Must be called after all flags are defined -// and before any are accessed by the program. +// Parse parses the command-line flags from os.Args[1:]. Must be called +// after all flags are defined and before flags are accessed by the program. func Parse() { - flags.args = os.Args[1:] - for flags.parseOne() { - } + // Ignore errors; commandLine is set for ExitOnError. + commandLine.Parse(os.Args[1:]) } -func init() { - flags = &allFlags{make(map[string]*Flag), make(map[string]*Flag), os.Args[1:]} +// The default set of command-line flags, parsed from os.Args. +var commandLine = NewFlagSet(os.Args[0], ExitOnError) + +// NewFlagSet returns a new, empty flag set with the specified name and +// error handling property. +func NewFlagSet(name string, errorHandling ErrorHandling) *FlagSet { + f := &FlagSet{ + name: name, + actual: make(map[string]*Flag), + formal: make(map[string]*Flag), + errorHandling: errorHandling, + } + f.Usage = func() { defaultUsage(f) } + return f } diff --git a/src/pkg/flag/flag_test.go b/src/pkg/flag/flag_test.go index 1e47d12e4..fbd706921 100644 --- a/src/pkg/flag/flag_test.go +++ b/src/pkg/flag/flag_test.go @@ -89,7 +89,7 @@ func TestEverything(t *testing.T) { func TestUsage(t *testing.T) { called := false ResetForTesting(func() { called = true }) - if ParseForTesting([]string{"a.out", "-x"}) { + if CommandLine().Parse([]string{"-x"}) == nil { t.Error("parse did not fail for unknown flag") } if !called { @@ -97,19 +97,17 @@ func TestUsage(t *testing.T) { } } -func TestParse(t *testing.T) { - ResetForTesting(func() { t.Error("bad parse") }) - boolFlag := Bool("bool", false, "bool value") - bool2Flag := Bool("bool2", false, "bool2 value") - intFlag := Int("int", 0, "int value") - int64Flag := Int64("int64", 0, "int64 value") - uintFlag := Uint("uint", 0, "uint value") - uint64Flag := Uint64("uint64", 0, "uint64 value") - stringFlag := String("string", "0", "string value") - float64Flag := Float64("float64", 0, "float64 value") +func testParse(f *FlagSet, t *testing.T) { + boolFlag := f.Bool("bool", false, "bool value") + bool2Flag := f.Bool("bool2", false, "bool2 value") + intFlag := f.Int("int", 0, "int value") + int64Flag := f.Int64("int64", 0, "int64 value") + uintFlag := f.Uint("uint", 0, "uint value") + uint64Flag := f.Uint64("uint64", 0, "uint64 value") + stringFlag := f.String("string", "0", "string value") + float64Flag := f.Float64("float64", 0, "float64 value") extra := "one-extra-argument" args := []string{ - "a.out", "-bool", "-bool2=true", "--int", "22", @@ -120,8 +118,8 @@ func TestParse(t *testing.T) { "-float64", "2718e28", extra, } - if !ParseForTesting(args) { - t.Fatal("parse failed") + if err := f.Parse(args); err != nil { + t.Fatal(err) } if *boolFlag != true { t.Error("bool flag should be true, is ", *boolFlag) @@ -147,14 +145,23 @@ func TestParse(t *testing.T) { if *float64Flag != 2718e28 { t.Error("float64 flag should be 2718e28, is ", *float64Flag) } - if len(Args()) != 1 { - t.Error("expected one argument, got", len(Args())) - } else if Args()[0] != extra { - t.Errorf("expected argument %q got %q", extra, Args()[0]) + if len(f.Args()) != 1 { + t.Error("expected one argument, got", len(f.Args())) + } else if f.Args()[0] != extra { + t.Errorf("expected argument %q got %q", extra, f.Args()[0]) } } -// Declare a user-defined flag. +func TestParse(t *testing.T) { + ResetForTesting(func() { t.Error("bad parse") }) + testParse(CommandLine(), t) +} + +func TestFlagSetParse(t *testing.T) { + testParse(NewFlagSet("test", ContinueOnError), t) +} + +// Declare a user-defined flag type. type flagVar []string func (f *flagVar) String() string { @@ -167,11 +174,11 @@ func (f *flagVar) Set(value string) bool { } func TestUserDefined(t *testing.T) { - ResetForTesting(func() { t.Fatal("bad parse") }) + flags := NewFlagSet("test", ContinueOnError) var v flagVar - Var(&v, "v", "usage") - if !ParseForTesting([]string{"a.out", "-v", "1", "-v", "2", "-v=3"}) { - t.Error("parse failed") + flags.Var(&v, "v", "usage") + if err := flags.Parse([]string{"-v", "1", "-v", "2", "-v=3"}); err != nil { + t.Error(err) } if len(v) != 3 { t.Fatal("expected 3 args; got ", len(v)) @@ -182,13 +189,17 @@ func TestUserDefined(t *testing.T) { } } +// This tests that one can reset the flags. This still works but not well, and is +// superseded by FlagSet. func TestChangingArgs(t *testing.T) { ResetForTesting(func() { t.Fatal("bad parse") }) oldArgs := os.Args defer func() { os.Args = oldArgs }() os.Args = []string{"cmd", "-before", "subcmd", "-after", "args"} before := Bool("before", false, "") - Parse() + if err := CommandLine().Parse(os.Args[1:]); err != nil { + t.Fatal(err) + } cmd := Arg(0) os.Args = Args() after := Bool("after", false, "") diff --git a/src/pkg/fmt/doc.go b/src/pkg/fmt/doc.go index e4d4f1844..79fe5758c 100644 --- a/src/pkg/fmt/doc.go +++ b/src/pkg/fmt/doc.go @@ -25,9 +25,10 @@ %c the character represented by the corresponding Unicode code point %d base 10 %o base 8 + %q a single-quoted character literal safely escaped with Go syntax. %x base 16, with lower-case letters for a-f %X base 16, with upper-case letters for A-F - %U Unicode format: U+1234; same as "U+%0.4X" + %U Unicode format: U+1234; same as "U+%04X" Floating-point and complex constituents: %b decimalless scientific notation with exponent a power of two, in the manner of strconv.Ftoa32, e.g. -123456p-78 @@ -134,10 +135,10 @@ The formats behave analogously to those of Printf with the following exceptions: - %p is not implemented - %T is not implemented - %e %E %f %F %g %g are all equivalent and scan any floating point or complex value - %s and %v on strings scan a space-delimited token + %p is not implemented + %T is not implemented + %e %E %f %F %g %G are all equivalent and scan any floating point or complex value + %s and %v on strings scan a space-delimited token The familiar base-setting prefixes 0 (octal) and 0x (hexadecimal) are accepted when scanning integers without a diff --git a/src/pkg/fmt/fmt_test.go b/src/pkg/fmt/fmt_test.go index b3c0c5abe..122b9516b 100644 --- a/src/pkg/fmt/fmt_test.go +++ b/src/pkg/fmt/fmt_test.go @@ -132,12 +132,23 @@ var fmttests = []struct { {"%q", `"`, `"\""`}, {"%q", "\a\b\f\r\n\t\v", `"\a\b\f\r\n\t\v"`}, {"%q", "abc\xffdef", `"abc\xffdef"`}, - {"%q", "\u263a", `"\u263a"`}, + {"%q", "\u263a", `"☺"`}, {"%q", "\U0010ffff", `"\U0010ffff"`}, + // escaped characters + {"%q", 'x', `'x'`}, + {"%q", 0, `'\x00'`}, + {"%q", '\n', `'\n'`}, + {"%q", '\u0e00', `'\u0e00'`}, // not a printable rune. + {"%q", '\U000c2345', `'\U000c2345'`}, // not a printable rune. + {"%q", int64(0x7FFFFFFF), `%!q(int64=2147483647)`}, + {"%q", uint64(0xFFFFFFFF), `%!q(uint64=4294967295)`}, + {"%q", '"', `'"'`}, + {"%q", '\'', `'\''`}, + // width {"%5s", "abc", " abc"}, - {"%2s", "\u263a", " \u263a"}, + {"%2s", "\u263a", " ☺"}, {"%-5s", "abc", "abc "}, {"%-8q", "abc", `"abc" `}, {"%05s", "abc", "00abc"}, @@ -147,9 +158,9 @@ var fmttests = []struct { {"%.5s", "日本語日本語", "日本語日本"}, {"%.5s", []byte("日本語日本語"), "日本語日本"}, {"%.5q", "abcdefghijklmnopqrstuvwxyz", `"abcde"`}, - {"%.3q", "日本語日本語", `"\u65e5\u672c\u8a9e"`}, - {"%.3q", []byte("日本語日本語"), `"\u65e5\u672c\u8a9e"`}, - {"%10.1q", "日本語日本語", ` "\u65e5"`}, + {"%.3q", "日本語日本語", `"日本語"`}, + {"%.3q", []byte("日本語日本語"), `"日本語"`}, + {"%10.1q", "日本語日本語", ` "日"`}, // integers {"%d", 12345, "12345"}, diff --git a/src/pkg/fmt/format.go b/src/pkg/fmt/format.go index f9d2b4fca..5dcfb9677 100644 --- a/src/pkg/fmt/format.go +++ b/src/pkg/fmt/format.go @@ -296,6 +296,13 @@ func (f *fmt) fmt_q(s string) { f.padString(quoted) } +// fmt_qc formats the integer as a single-quoted, escaped Go character constant. +// If the character is not valid Unicode, it will print '\ufffd'. +func (f *fmt) fmt_qc(c int64) { + quoted := strconv.QuoteRune(int(c)) + f.padString(quoted) +} + // floating-point func doPrec(f *fmt, def int) int { diff --git a/src/pkg/fmt/print.go b/src/pkg/fmt/print.go index 10e0fe7c8..c18a8ea38 100644 --- a/src/pkg/fmt/print.go +++ b/src/pkg/fmt/print.go @@ -9,6 +9,7 @@ import ( "io" "os" "reflect" + "unicode" "utf8" ) @@ -41,7 +42,7 @@ type State interface { Precision() (prec int, ok bool) // Flag returns whether the flag c, a character, has been set. - Flag(int) bool + Flag(c int) bool } // Formatter is the interface implemented by values with a custom formatter. @@ -332,6 +333,12 @@ func (p *pp) fmtInt64(v int64, verb int, value interface{}) { p.fmt.integer(v, 10, signed, ldigits) case 'o': p.fmt.integer(v, 8, signed, ldigits) + case 'q': + if 0 <= v && v <= unicode.MaxRune { + p.fmt.fmt_qc(v) + } else { + p.badVerb(verb, value) + } case 'x': p.fmt.integer(v, 16, signed, ldigits) case 'U': @@ -385,6 +392,12 @@ func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}) { } case 'o': p.fmt.integer(int64(v), 8, unsigned, ldigits) + case 'q': + if 0 <= v && v <= unicode.MaxRune { + p.fmt.fmt_qc(int64(v)) + } else { + p.badVerb(verb, value) + } case 'x': p.fmt.integer(int64(v), 16, unsigned, ldigits) case 'X': diff --git a/src/pkg/fmt/scan.go b/src/pkg/fmt/scan.go index 42bc52c92..dd8548ceb 100644 --- a/src/pkg/fmt/scan.go +++ b/src/pkg/fmt/scan.go @@ -231,6 +231,7 @@ func (s *ss) UnreadRune() os.Error { } else { s.peekRune = s.prevRune } + s.prevRune = -1 s.count-- return nil } @@ -457,6 +458,14 @@ func (s *ss) peek(ok string) bool { return strings.IndexRune(ok, rune) >= 0 } +func (s *ss) notEOF() { + // Guarantee there is data to be read. + if rune := s.getRune(); rune == eof { + panic(os.EOF) + } + s.UnreadRune() +} + // accept checks the next rune in the input. If it's a byte (sic) in the string, it puts it in the // buffer and returns true. Otherwise it return false. func (s *ss) accept(ok string) bool { @@ -476,11 +485,13 @@ func (s *ss) okVerb(verb int, okVerbs, typ string) bool { // scanBool returns the value of the boolean represented by the next token. func (s *ss) scanBool(verb int) bool { + s.skipSpace(false) + s.notEOF() if !s.okVerb(verb, "tv", "boolean") { return false } // Syntax-checking a boolean is annoying. We're not fastidious about case. - switch s.mustReadRune() { + switch s.getRune() { case '0': return false case '1': @@ -531,6 +542,7 @@ func (s *ss) getBase(verb int) (base int, digits string) { // scanNumber returns the numerical string with specified digits starting here. func (s *ss) scanNumber(digits string, haveDigits bool) string { + s.notEOF() if !haveDigits && !s.accept(digits) { s.errorString("expected integer") } @@ -541,7 +553,8 @@ func (s *ss) scanNumber(digits string, haveDigits bool) string { // scanRune returns the next rune value in the input. func (s *ss) scanRune(bitSize int) int64 { - rune := int64(s.mustReadRune()) + s.notEOF() + rune := int64(s.getRune()) n := uint(bitSize) x := (rune << (64 - n)) >> (64 - n) if x != rune { @@ -575,6 +588,7 @@ func (s *ss) scanInt(verb int, bitSize int) int64 { return s.scanRune(bitSize) } s.skipSpace(false) + s.notEOF() base, digits := s.getBase(verb) haveDigits := false if verb == 'U' { @@ -607,6 +621,7 @@ func (s *ss) scanUint(verb int, bitSize int) uint64 { return uint64(s.scanRune(bitSize)) } s.skipSpace(false) + s.notEOF() base, digits := s.getBase(verb) haveDigits := false if verb == 'U' { @@ -727,6 +742,7 @@ func (s *ss) scanComplex(verb int, n int) complex128 { return 0 } s.skipSpace(false) + s.notEOF() sreal, simag := s.complexTokens() real := s.convertFloat(sreal, n/2) imag := s.convertFloat(simag, n/2) @@ -740,6 +756,7 @@ func (s *ss) convertString(verb int) (str string) { return "" } s.skipSpace(false) + s.notEOF() switch verb { case 'q': str = s.quotedString() @@ -748,16 +765,13 @@ func (s *ss) convertString(verb int) (str string) { default: str = string(s.token(true, notSpace)) // %s and %v just return the next word } - // Empty strings other than with %q are not OK. - if len(str) == 0 && verb != 'q' && s.maxWid > 0 { - s.errorString("Scan: no data for string") - } return } // quotedString returns the double- or back-quoted string represented by the next input characters. func (s *ss) quotedString() string { - quote := s.mustReadRune() + s.notEOF() + quote := s.getRune() switch quote { case '`': // Back-quoted: Anything goes until EOF or back quote. @@ -827,6 +841,7 @@ func (s *ss) hexByte() (b byte, ok bool) { // hexString returns the space-delimited hexpair-encoded string. func (s *ss) hexString() string { + s.notEOF() for { b, ok := s.hexByte() if !ok { @@ -860,6 +875,7 @@ func (s *ss) scanOne(verb int, field interface{}) { } return } + switch v := field.(type) { case *bool: *v = s.scanBool(verb) @@ -894,11 +910,13 @@ func (s *ss) scanOne(verb int, field interface{}) { case *float32: if s.okVerb(verb, floatVerbs, "float32") { s.skipSpace(false) + s.notEOF() *v = float32(s.convertFloat(s.floatToken(), 32)) } case *float64: if s.okVerb(verb, floatVerbs, "float64") { s.skipSpace(false) + s.notEOF() *v = s.convertFloat(s.floatToken(), 64) } case *string: @@ -936,6 +954,7 @@ func (s *ss) scanOne(verb int, field interface{}) { } case reflect.Float32, reflect.Float64: s.skipSpace(false) + s.notEOF() v.SetFloat(s.convertFloat(s.floatToken(), v.Type().Bits())) case reflect.Complex64, reflect.Complex128: v.SetComplex(s.scanComplex(verb, v.Type().Bits())) @@ -946,13 +965,13 @@ func (s *ss) scanOne(verb int, field interface{}) { } } -// errorHandler turns local panics into error returns. EOFs are benign. +// errorHandler turns local panics into error returns. func errorHandler(errp *os.Error) { if e := recover(); e != nil { if se, ok := e.(scanError); ok { // catch local error - if se.err != os.EOF { - *errp = se.err - } + *errp = se.err + } else if eof, ok := e.(os.Error); ok && eof == os.EOF { // out of input + *errp = eof } else { panic(e) } diff --git a/src/pkg/fmt/scan_test.go b/src/pkg/fmt/scan_test.go index da13eb2d1..a4de8adb1 100644 --- a/src/pkg/fmt/scan_test.go +++ b/src/pkg/fmt/scan_test.go @@ -660,6 +660,68 @@ func TestEOF(t *testing.T) { } } +// Verify that we see an EOF error if we run out of input. +// This was a buglet: we used to get "expected integer". +func TestEOFAtEndOfInput(t *testing.T) { + var i, j int + n, err := Sscanf("23", "%d %d", &i, &j) + if n != 1 || i != 23 { + t.Errorf("Sscanf expected one value of 23; got %d %d", n, i) + } + if err != os.EOF { + t.Errorf("Sscanf expected EOF; got %q", err) + } + n, err = Sscan("234", &i, &j) + if n != 1 || i != 234 { + t.Errorf("Sscan expected one value of 234; got %d %d", n, i) + } + if err != os.EOF { + t.Errorf("Sscan expected EOF; got %q", err) + } + // Trailing space is tougher. + n, err = Sscan("234 ", &i, &j) + if n != 1 || i != 234 { + t.Errorf("Sscan expected one value of 234; got %d %d", n, i) + } + if err != os.EOF { + t.Errorf("Sscan expected EOF; got %q", err) + } +} + +var eofTests = []struct { + format string + v interface{} +}{ + {"%s", &stringVal}, + {"%q", &stringVal}, + {"%x", &stringVal}, + {"%v", &stringVal}, + {"%v", &bytesVal}, + {"%v", &intVal}, + {"%v", &uintVal}, + {"%v", &boolVal}, + {"%v", &float32Val}, + {"%v", &complex64Val}, + {"%v", &renamedStringVal}, + {"%v", &renamedBytesVal}, + {"%v", &renamedIntVal}, + {"%v", &renamedUintVal}, + {"%v", &renamedBoolVal}, + {"%v", &renamedFloat32Val}, + {"%v", &renamedComplex64Val}, +} + +func TestEOFAllTypes(t *testing.T) { + for i, test := range eofTests { + if _, err := Sscanf("", test.format, test.v); err != os.EOF { + t.Errorf("#%d: %s %T not eof on empty string: %s", i, test.format, test.v, err) + } + if _, err := Sscanf(" ", test.format, test.v); err != os.EOF { + t.Errorf("#%d: %s %T not eof on trailing blanks: %s", i, test.format, test.v, err) + } + } +} + // Verify that, at least when using bufio, successive calls to Fscan do not lose runes. func TestUnreadRuneWithBufio(t *testing.T) { r := bufio.NewReader(strings.NewReader("123αb")) diff --git a/src/pkg/go/ast/ast.go b/src/pkg/go/ast/ast.go index 2fc1a6032..b1c7d4ab1 100644 --- a/src/pkg/go/ast/ast.go +++ b/src/pkg/go/ast/ast.go @@ -121,7 +121,7 @@ func (f *Field) End() token.Pos { // A FieldList represents a list of Fields, enclosed by parentheses or braces. type FieldList struct { Opening token.Pos // position of opening parenthesis/brace, if any - List []*Field // field list + List []*Field // field list; or nil Closing token.Pos // position of closing parenthesis/brace, if any } @@ -334,7 +334,7 @@ type ( // A FuncType node represents a function type. FuncType struct { Func token.Pos // position of "func" keyword - Params *FieldList // (incoming) parameters + Params *FieldList // (incoming) parameters; or nil Results *FieldList // (outgoing) results; or nil } @@ -515,10 +515,10 @@ type ( // An EmptyStmt node represents an empty statement. // The "position" of the empty statement is the position - // of the immediately preceeding semicolon. + // of the immediately preceding semicolon. // EmptyStmt struct { - Semicolon token.Pos // position of preceeding ";" + Semicolon token.Pos // position of preceding ";" } // A LabeledStmt node represents a labeled statement. @@ -596,7 +596,7 @@ type ( // An IfStmt node represents an if statement. IfStmt struct { If token.Pos // position of "if" keyword - Init Stmt // initalization statement; or nil + Init Stmt // initialization statement; or nil Cond Expr // condition Body *BlockStmt Else Stmt // else branch; or nil @@ -613,7 +613,7 @@ type ( // A SwitchStmt node represents an expression switch statement. SwitchStmt struct { Switch token.Pos // position of "switch" keyword - Init Stmt // initalization statement; or nil + Init Stmt // initialization statement; or nil Tag Expr // tag expression; or nil Body *BlockStmt // CaseClauses only } @@ -621,7 +621,7 @@ type ( // An TypeSwitchStmt node represents a type switch statement. TypeSwitchStmt struct { Switch token.Pos // position of "switch" keyword - Init Stmt // initalization statement; or nil + Init Stmt // initialization statement; or nil Assign Stmt // x := y.(type) or y.(type) Body *BlockStmt // CaseClauses only } @@ -643,7 +643,7 @@ type ( // A ForStmt represents a for statement. ForStmt struct { For token.Pos // position of "for" keyword - Init Stmt // initalization statement; or nil + Init Stmt // initialization statement; or nil Cond Expr // condition; or nil Post Stmt // post iteration statement; or nil Body *BlockStmt @@ -945,10 +945,10 @@ func (f *File) End() token.Pos { // collectively building a Go package. // type Package struct { - Name string // package name - Scope *Scope // package scope - Imports map[string]*Scope // map of import path -> package scope across all files - Files map[string]*File // Go source files by filename + Name string // package name + Scope *Scope // package scope across all files + Imports map[string]*Object // map of package id -> package object + Files map[string]*File // Go source files by filename } diff --git a/src/pkg/go/ast/filter.go b/src/pkg/go/ast/filter.go index 090d08d34..0907fd53d 100644 --- a/src/pkg/go/ast/filter.go +++ b/src/pkg/go/ast/filter.go @@ -21,24 +21,26 @@ func identListExports(list []*Ident) []*Ident { } -// isExportedType assumes that typ is a correct type. -func isExportedType(typ Expr) bool { - switch t := typ.(type) { +// fieldName assumes that x is the type of an anonymous field and +// returns the corresponding field name. If x is not an acceptable +// anonymous field, the result is nil. +// +func fieldName(x Expr) *Ident { + switch t := x.(type) { case *Ident: - return t.IsExported() - case *ParenExpr: - return isExportedType(t.X) + return t case *SelectorExpr: - // assume t.X is a typename - return t.Sel.IsExported() + if _, ok := t.X.(*Ident); ok { + return t.Sel + } case *StarExpr: - return isExportedType(t.X) + return fieldName(t.X) } - return false + return nil } -func fieldListExports(fields *FieldList, incomplete *bool) { +func fieldListExports(fields *FieldList) (removedFields bool) { if fields == nil { return } @@ -53,12 +55,13 @@ func fieldListExports(fields *FieldList, incomplete *bool) { // fields, so this is not absolutely correct. // However, this cannot be done w/o complete // type information.) - exported = isExportedType(f.Type) + name := fieldName(f.Type) + exported = name != nil && name.IsExported() } else { n := len(f.Names) f.Names = identListExports(f.Names) if len(f.Names) < n { - *incomplete = true + removedFields = true } exported = len(f.Names) > 0 } @@ -69,9 +72,10 @@ func fieldListExports(fields *FieldList, incomplete *bool) { } } if j < len(list) { - *incomplete = true + removedFields = true } fields.List = list[0:j] + return } @@ -90,12 +94,16 @@ func typeExports(typ Expr) { case *ArrayType: typeExports(t.Elt) case *StructType: - fieldListExports(t.Fields, &t.Incomplete) + if fieldListExports(t.Fields) { + t.Incomplete = true + } case *FuncType: paramListExports(t.Params) paramListExports(t.Results) case *InterfaceType: - fieldListExports(t.Methods, &t.Incomplete) + if fieldListExports(t.Methods) { + t.Incomplete = true + } case *MapType: typeExports(t.Key) typeExports(t.Value) @@ -206,13 +214,60 @@ func filterIdentList(list []*Ident, f Filter) []*Ident { } +func filterFieldList(fields *FieldList, filter Filter) (removedFields bool) { + if fields == nil { + return false + } + list := fields.List + j := 0 + for _, f := range list { + keepField := false + if len(f.Names) == 0 { + // anonymous field + name := fieldName(f.Type) + keepField = name != nil && filter(name.Name) + } else { + n := len(f.Names) + f.Names = filterIdentList(f.Names, filter) + if len(f.Names) < n { + removedFields = true + } + keepField = len(f.Names) > 0 + } + if keepField { + list[j] = f + j++ + } + } + if j < len(list) { + removedFields = true + } + fields.List = list[0:j] + return +} + + func filterSpec(spec Spec, f Filter) bool { switch s := spec.(type) { case *ValueSpec: s.Names = filterIdentList(s.Names, f) return len(s.Names) > 0 case *TypeSpec: - return f(s.Name.Name) + if f(s.Name.Name) { + return true + } + switch t := s.Type.(type) { + case *StructType: + if filterFieldList(t.Fields, f) { + t.Incomplete = true + } + return len(t.Fields.List) > 0 + case *InterfaceType: + if filterFieldList(t.Methods, f) { + t.Incomplete = true + } + return len(t.Methods.List) > 0 + } } return false } @@ -230,7 +285,14 @@ func filterSpecList(list []Spec, f Filter) []Spec { } -func filterDecl(decl Decl, f Filter) bool { +// FilterDecl trims the AST for a Go declaration in place by removing +// all names (including struct field and interface method names, but +// not from parameter lists) that don't pass through the filter f. +// +// FilterDecl returns true if there are any declared names left after +// filtering; it returns false otherwise. +// +func FilterDecl(decl Decl, f Filter) bool { switch d := decl.(type) { case *GenDecl: d.Specs = filterSpecList(d.Specs, f) @@ -243,10 +305,10 @@ func filterDecl(decl Decl, f Filter) bool { // FilterFile trims the AST for a Go file in place by removing all -// names from top-level declarations (but not from parameter lists -// or inside types) that don't pass through the filter f. If the -// declaration is empty afterwards, the declaration is removed from -// the AST. +// names from top-level declarations (including struct field and +// interface method names, but not from parameter lists) that don't +// pass through the filter f. If the declaration is empty afterwards, +// the declaration is removed from the AST. // The File.comments list is not changed. // // FilterFile returns true if there are any top-level declarations @@ -255,7 +317,7 @@ func filterDecl(decl Decl, f Filter) bool { func FilterFile(src *File, f Filter) bool { j := 0 for _, d := range src.Decls { - if filterDecl(d, f) { + if FilterDecl(d, f) { src.Decls[j] = d j++ } @@ -266,10 +328,10 @@ func FilterFile(src *File, f Filter) bool { // FilterPackage trims the AST for a Go package in place by removing all -// names from top-level declarations (but not from parameter lists -// or inside types) that don't pass through the filter f. If the -// declaration is empty afterwards, the declaration is removed from -// the AST. +// names from top-level declarations (including struct field and +// interface method names, but not from parameter lists) that don't +// pass through the filter f. If the declaration is empty afterwards, +// the declaration is removed from the AST. // The pkg.Files list is not changed, so that file names and top-level // package comments don't get lost. // diff --git a/src/pkg/go/ast/resolve.go b/src/pkg/go/ast/resolve.go index fddc3baab..ecd2e8a7c 100644 --- a/src/pkg/go/ast/resolve.go +++ b/src/pkg/go/ast/resolve.go @@ -11,6 +11,7 @@ import ( "go/scanner" "go/token" "os" + "strconv" ) @@ -57,11 +58,16 @@ func resolve(scope *Scope, ident *Ident) bool { } -// NewPackage uses an Importer to resolve imports. Given an importPath, -// an importer returns the imported package's name, its scope of exported -// objects, and an error, if any. -// -type Importer func(path string) (name string, scope *Scope, err os.Error) +// An Importer resolves import paths to package Objects. +// The imports map records the packages already imported, +// indexed by package id (canonical import path). +// An Importer must determine the canonical import path and +// check the map to see if it is already present in the imports map. +// If so, the Importer can return the map entry. Otherwise, the +// Importer should load the package data for the given path into +// a new *Object (pkg), record pkg in the imports map, and then +// return pkg. +type Importer func(imports map[string]*Object, path string) (pkg *Object, err os.Error) // NewPackage creates a new Package node from a set of File nodes. It resolves @@ -70,7 +76,7 @@ type Importer func(path string) (name string, scope *Scope, err os.Error) // used to resolve identifiers not declared in any of the package files. Any // remaining unresolved identifiers are reported as undeclared. If the files // belong to different packages, one package name is selected and files with -// different package name are reported and then ignored. +// different package names are reported and then ignored. // The result is a package node and a scanner.ErrorList if there were errors. // func NewPackage(fset *token.FileSet, files map[string]*File, importer Importer, universe *Scope) (*Package, os.Error) { @@ -96,14 +102,8 @@ func NewPackage(fset *token.FileSet, files map[string]*File, importer Importer, } } - // imports maps import paths to package names and scopes - // TODO(gri): Eventually we like to get to the import scope from - // a package object. Then we can have a map path -> Obj. - type importedPkg struct { - name string - scope *Scope - } - imports := make(map[string]*importedPkg) + // package global mapping of imported package ids to package objects + imports := make(map[string]*Object) // complete file scopes with imports and resolve identifiers for _, file := range files { @@ -117,42 +117,41 @@ func NewPackage(fset *token.FileSet, files map[string]*File, importer Importer, importErrors := false fileScope := NewScope(pkgScope) for _, spec := range file.Imports { - // add import to global map of imports - path := string(spec.Path.Value) - path = path[1 : len(path)-1] // strip ""'s - pkg := imports[path] - if pkg == nil { - if importer == nil { - importErrors = true - continue - } - name, scope, err := importer(path) - if err != nil { - p.errorf(spec.Path.Pos(), "could not import %s (%s)", path, err) - importErrors = true - continue - } - pkg = &importedPkg{name, scope} - imports[path] = pkg - // TODO(gri) If a local package name != "." is provided, - // global identifier resolution could proceed even if the - // import failed. Consider adjusting the logic here a bit. + if importer == nil { + importErrors = true + continue + } + path, _ := strconv.Unquote(string(spec.Path.Value)) + pkg, err := importer(imports, path) + if err != nil { + p.errorf(spec.Path.Pos(), "could not import %s (%s)", path, err) + importErrors = true + continue } + // TODO(gri) If a local package name != "." is provided, + // global identifier resolution could proceed even if the + // import failed. Consider adjusting the logic here a bit. + // local name overrides imported package name - name := pkg.name + name := pkg.Name if spec.Name != nil { name = spec.Name.Name } + // add import to file scope if name == "." { // merge imported scope with file scope - for _, obj := range pkg.scope.Objects { + for _, obj := range pkg.Data.(*Scope).Objects { p.declare(fileScope, pkgScope, obj) } } else { // declare imported package object in file scope + // (do not re-use pkg in the file scope but create + // a new object instead; the Decl field is different + // for different files) obj := NewObj(Pkg, name) obj.Decl = spec + obj.Data = pkg.Data p.declare(fileScope, pkgScope, obj) } } @@ -161,8 +160,8 @@ func NewPackage(fset *token.FileSet, files map[string]*File, importer Importer, if importErrors { // don't use the universe scope without correct imports // (objects in the universe may be shadowed by imports; - // with missing imports identifiers might get resolved - // wrongly) + // with missing imports, identifiers might get resolved + // incorrectly to universe objects) pkgScope.Outer = nil } i := 0 @@ -178,11 +177,5 @@ func NewPackage(fset *token.FileSet, files map[string]*File, importer Importer, pkgScope.Outer = universe // reset universe scope } - // collect all import paths and respective package scopes - importedScopes := make(map[string]*Scope) - for path, pkg := range imports { - importedScopes[path] = pkg.scope - } - - return &Package{pkgName, pkgScope, importedScopes, files}, p.GetError(scanner.Sorted) + return &Package{pkgName, pkgScope, imports, files}, p.GetError(scanner.Sorted) } diff --git a/src/pkg/go/ast/scope.go b/src/pkg/go/ast/scope.go index 830d88aef..b966f786f 100644 --- a/src/pkg/go/ast/scope.go +++ b/src/pkg/go/ast/scope.go @@ -70,13 +70,24 @@ func (s *Scope) String() string { // ---------------------------------------------------------------------------- // Objects +// TODO(gri) Consider replacing the Object struct with an interface +// and a corresponding set of object implementations. + // An Object describes a named language entity such as a package, // constant, type, variable, function (incl. methods), or label. // +// The Data fields contains object-specific data: +// +// Kind Data type Data value +// Pkg *Scope package scope +// Con int iota for the respective declaration +// Con != nil constant value +// type Object struct { Kind ObjKind Name string // declared name Decl interface{} // corresponding Field, XxxSpec, FuncDecl, or LabeledStmt; or nil + Data interface{} // object-specific data; or nil Type interface{} // place holder for type information; may be nil } diff --git a/src/pkg/go/build/Makefile b/src/pkg/go/build/Makefile new file mode 100644 index 000000000..4411940ae --- /dev/null +++ b/src/pkg/go/build/Makefile @@ -0,0 +1,22 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=go/build +GOFILES=\ + build.go\ + dir.go\ + path.go\ + syslist.go\ + +CLEANFILES+=syslist.go + +include ../../../Make.pkg + +syslist.go: ../../../Make.inc Makefile + echo '// Generated automatically by make.' >$@ + echo 'package build' >>$@ + echo 'const goosList = "$(GOOS_LIST)"' >>$@ + echo 'const goarchList = "$(GOARCH_LIST)"' >>$@ diff --git a/src/pkg/go/build/build.go b/src/pkg/go/build/build.go new file mode 100644 index 000000000..3cb8efe47 --- /dev/null +++ b/src/pkg/go/build/build.go @@ -0,0 +1,272 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package build provides tools for building Go packages. +package build + +import ( + "bytes" + "exec" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" +) + +func (d *DirInfo) Build(targ string) ([]*Cmd, os.Error) { + b := &build{obj: "_obj/"} + + goarch := runtime.GOARCH + if g := os.Getenv("GOARCH"); g != "" { + goarch = g + } + var err os.Error + b.arch, err = ArchChar(goarch) + if err != nil { + return nil, err + } + + var gofiles = d.GoFiles // .go files to be built with gc + var ofiles []string // *.GOARCH files to be linked or packed + + // make build directory + b.mkdir(b.obj) + + // cgo + if len(d.CgoFiles) > 0 { + outGo, outObj := b.cgo(d.CgoFiles) + gofiles = append(gofiles, outGo...) + ofiles = append(ofiles, outObj...) + } + + // compile + if len(gofiles) > 0 { + ofile := b.obj + "_go_." + b.arch + b.gc(ofile, gofiles...) + ofiles = append(ofiles, ofile) + } + + // assemble + for _, sfile := range d.SFiles { + ofile := b.obj + sfile[:len(sfile)-1] + b.arch + b.asm(ofile, sfile) + ofiles = append(ofiles, ofile) + } + + if len(ofiles) == 0 { + return nil, os.NewError("make: no object files to build") + } + + if d.IsCommand() { + b.ld(targ, ofiles...) + } else { + b.gopack(targ, ofiles...) + } + + return b.cmds, nil +} + +type Cmd struct { + Args []string // command-line + Stdout string // write standard output to this file, "" is passthrough + Input []string // file paths (dependencies) + Output []string // file paths +} + +func (c *Cmd) String() string { + return strings.Join(c.Args, " ") +} + +func (c *Cmd) Run(dir string) os.Error { + out := new(bytes.Buffer) + cmd := exec.Command(c.Args[0], c.Args[1:]...) + cmd.Dir = dir + cmd.Stdout = out + cmd.Stderr = out + if c.Stdout != "" { + f, err := os.Create(filepath.Join(dir, c.Stdout)) + if err != nil { + return err + } + defer f.Close() + cmd.Stdout = f + } + if err := cmd.Run(); err != nil { + return fmt.Errorf("command %q: %v\n%v", c, err, out) + } + return nil +} + +func (c *Cmd) Clean(dir string) (err os.Error) { + for _, fn := range c.Output { + if e := os.RemoveAll(fn); err == nil { + err = e + } + } + return +} + +// ArchChar returns the architecture character for the given goarch. +// For example, ArchChar("amd64") returns "6". +func ArchChar(goarch string) (string, os.Error) { + switch goarch { + case "386": + return "8", nil + case "amd64": + return "6", nil + case "arm": + return "5", nil + } + return "", os.NewError("unsupported GOARCH " + goarch) +} + +type build struct { + cmds []*Cmd + obj string + arch string +} + +func (b *build) add(c Cmd) { + b.cmds = append(b.cmds, &c) +} + +func (b *build) mkdir(name string) { + b.add(Cmd{ + Args: []string{"mkdir", "-p", name}, + Output: []string{name}, + }) +} + +func (b *build) gc(ofile string, gofiles ...string) { + gc := b.arch + "g" + args := append([]string{gc, "-o", ofile}, gcImportArgs...) + args = append(args, gofiles...) + b.add(Cmd{ + Args: args, + Input: gofiles, + Output: []string{ofile}, + }) +} + +func (b *build) asm(ofile string, sfile string) { + asm := b.arch + "a" + b.add(Cmd{ + Args: []string{asm, "-o", ofile, sfile}, + Input: []string{sfile}, + Output: []string{ofile}, + }) +} + +func (b *build) ld(targ string, ofiles ...string) { + ld := b.arch + "l" + args := append([]string{ld, "-o", targ}, ldImportArgs...) + args = append(args, ofiles...) + b.add(Cmd{ + Args: args, + Input: ofiles, + Output: []string{targ}, + }) +} + +func (b *build) gopack(targ string, ofiles ...string) { + b.add(Cmd{ + Args: append([]string{"gopack", "grc", targ}, ofiles...), + Input: ofiles, + Output: []string{targ}, + }) +} + +func (b *build) cc(ofile string, cfiles ...string) { + cc := b.arch + "c" + dir := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH) + inc := filepath.Join(runtime.GOROOT(), "pkg", dir) + args := []string{cc, "-FVw", "-I", inc, "-o", ofile} + b.add(Cmd{ + Args: append(args, cfiles...), + Input: cfiles, + Output: []string{ofile}, + }) +} + +func (b *build) gccCompile(ofile, cfile string) { + b.add(Cmd{ + Args: gccArgs(b.arch, "-o", ofile, "-c", cfile), + Input: []string{cfile}, + Output: []string{ofile}, + }) +} + +func (b *build) gccLink(ofile string, ofiles ...string) { + b.add(Cmd{ + Args: append(gccArgs(b.arch, "-o", ofile), ofiles...), + Input: ofiles, + Output: []string{ofile}, + }) +} + +func gccArgs(arch string, args ...string) []string { + // TODO(adg): HOST_CC + m := "-m32" + if arch == "6" { + m = "-m64" + } + return append([]string{"gcc", m, "-I", ".", "-g", "-fPIC", "-O2"}, args...) +} + +func (b *build) cgo(cgofiles []string) (outGo, outObj []string) { + // cgo + // TODO(adg): CGOPKGPATH + // TODO(adg): CGO_FLAGS + gofiles := []string{b.obj + "_cgo_gotypes.go"} + cfiles := []string{b.obj + "_cgo_main.c", b.obj + "_cgo_export.c"} + for _, fn := range cgofiles { + f := b.obj + fn[:len(fn)-2] + gofiles = append(gofiles, f+"cgo1.go") + cfiles = append(cfiles, f+"cgo2.c") + } + defunC := b.obj + "_cgo_defun.c" + output := append([]string{defunC}, gofiles...) + output = append(output, cfiles...) + b.add(Cmd{ + Args: append([]string{"cgo", "--"}, cgofiles...), + Input: cgofiles, + Output: output, + }) + outGo = append(outGo, gofiles...) + + // cc _cgo_defun.c + defunObj := b.obj + "_cgo_defun." + b.arch + b.cc(defunObj, defunC) + outObj = append(outObj, defunObj) + + // gcc + linkobj := make([]string, 0, len(cfiles)) + for _, cfile := range cfiles { + ofile := cfile[:len(cfile)-1] + "o" + b.gccCompile(ofile, cfile) + linkobj = append(linkobj, ofile) + if !strings.HasSuffix(ofile, "_cgo_main.o") { + outObj = append(outObj, ofile) + } + } + dynObj := b.obj + "_cgo1_.o" + b.gccLink(dynObj, linkobj...) + + // cgo -dynimport + importC := b.obj + "_cgo_import.c" + b.add(Cmd{ + Args: []string{"cgo", "-dynimport", dynObj}, + Stdout: importC, + Input: []string{dynObj}, + Output: []string{importC}, + }) + + // cc _cgo_import.ARCH + importObj := b.obj + "_cgo_import." + b.arch + b.cc(importObj, importC) + outObj = append(outObj, importObj) + + return +} diff --git a/src/pkg/go/build/build_test.go b/src/pkg/go/build/build_test.go new file mode 100644 index 000000000..c543eddbd --- /dev/null +++ b/src/pkg/go/build/build_test.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +var buildDirs = []string{ + "pkg/path", + "cmd/gofix", + "pkg/big", + "pkg/go/build/cgotest", +} + +func TestBuild(t *testing.T) { + out, err := filepath.Abs("_test/out") + if err != nil { + t.Fatal(err) + } + for _, d := range buildDirs { + if runtime.GOARCH == "arm" && strings.Contains(d, "/cgo") { + // no cgo for arm, yet. + continue + } + dir := filepath.Join(runtime.GOROOT(), "src", d) + testBuild(t, dir, out) + } +} + +func testBuild(t *testing.T, dir, targ string) { + d, err := ScanDir(dir, true) + if err != nil { + t.Error(err) + return + } + defer os.Remove(targ) + cmds, err := d.Build(targ) + if err != nil { + t.Error(err) + return + } + for _, c := range cmds { + t.Log("Run:", c) + err = c.Run(dir) + if err != nil { + t.Error(c, err) + return + } + } +} diff --git a/src/pkg/go/build/cgotest/file.go b/src/pkg/go/build/cgotest/file.go new file mode 100644 index 000000000..3b2a2e7d9 --- /dev/null +++ b/src/pkg/go/build/cgotest/file.go @@ -0,0 +1,45 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +A trivial example of wrapping a C library in Go. +For a more complex example and explanation, +see ../gmp/gmp.go. +*/ + +package stdio + +/* +#include <stdio.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <errno.h> + +char* greeting = "hello, world"; +*/ +import "C" +import "unsafe" + +type File C.FILE + +// TODO(brainman): uncomment once stdout and stderr references are working on Windows. +//var Stdout = (*File)(C.stdout) +//var Stderr = (*File)(C.stderr) + +// Test reference to library symbol. +// Stdout and stderr are too special to be a reliable test. +var myerr = C.sys_errlist + +func (f *File) WriteString(s string) { + p := C.CString(s) + C.fputs(p, (*C.FILE)(f)) + C.free(unsafe.Pointer(p)) + f.Flush() +} + +func (f *File) Flush() { + C.fflush((*C.FILE)(f)) +} + +var Greeting = C.GoString(C.greeting) diff --git a/src/pkg/go/build/dir.go b/src/pkg/go/build/dir.go new file mode 100644 index 000000000..77e80bff0 --- /dev/null +++ b/src/pkg/go/build/dir.go @@ -0,0 +1,173 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "runtime" +) + +type DirInfo struct { + GoFiles []string // .go files in dir (excluding CgoFiles) + CgoFiles []string // .go files that import "C" + CFiles []string // .c files in dir + SFiles []string // .s files in dir + Imports []string // All packages imported by goFiles + PkgName string // Name of package in dir +} + +func (d *DirInfo) IsCommand() bool { + return d.PkgName == "main" +} + +// ScanDir returns a structure with details about the Go content found +// in the given directory. The file lists exclude: +// +// - files in package main (unless allowMain is true) +// - files in package documentation +// - files ending in _test.go +// - files starting with _ or . +// +// Only files that satisfy the goodOSArch function are included. +func ScanDir(dir string, allowMain bool) (info *DirInfo, err os.Error) { + f, err := os.Open(dir) + if err != nil { + return nil, err + } + dirs, err := f.Readdir(-1) + f.Close() + if err != nil { + return nil, err + } + + var di DirInfo + imported := make(map[string]bool) + pkgName := "" + fset := token.NewFileSet() + for i := range dirs { + d := &dirs[i] + if strings.HasPrefix(d.Name, "_") || + strings.HasPrefix(d.Name, ".") { + continue + } + if !goodOSArch(d.Name) { + continue + } + + switch filepath.Ext(d.Name) { + case ".go": + if strings.HasSuffix(d.Name, "_test.go") { + continue + } + case ".c": + di.CFiles = append(di.CFiles, d.Name) + continue + case ".s": + di.SFiles = append(di.SFiles, d.Name) + continue + default: + continue + } + + filename := filepath.Join(dir, d.Name) + pf, err := parser.ParseFile(fset, filename, nil, parser.ImportsOnly) + if err != nil { + return nil, err + } + s := string(pf.Name.Name) + if s == "main" && !allowMain { + continue + } + if s == "documentation" { + continue + } + if pkgName == "" { + pkgName = s + } else if pkgName != s { + // Only if all files in the directory are in package main + // do we return pkgName=="main". + // A mix of main and another package reverts + // to the original (allowMain=false) behaviour. + if s == "main" || pkgName == "main" { + return ScanDir(dir, false) + } + return nil, os.ErrorString("multiple package names in " + dir) + } + isCgo := false + for _, spec := range pf.Imports { + quoted := string(spec.Path.Value) + path, err := strconv.Unquote(quoted) + if err != nil { + log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted) + } + imported[path] = true + if path == "C" { + isCgo = true + } + } + if isCgo { + di.CgoFiles = append(di.CgoFiles, d.Name) + } else { + di.GoFiles = append(di.GoFiles, d.Name) + } + } + di.Imports = make([]string, len(imported)) + i := 0 + for p := range imported { + di.Imports[i] = p + i++ + } + return &di, nil +} + +// goodOSArch returns false if the filename contains a $GOOS or $GOARCH +// suffix which does not match the current system. +// The recognized filename formats are: +// +// name_$(GOOS).* +// name_$(GOARCH).* +// name_$(GOOS)_$(GOARCH).* +// +func goodOSArch(filename string) bool { + if dot := strings.Index(filename, "."); dot != -1 { + filename = filename[:dot] + } + l := strings.Split(filename, "_", -1) + n := len(l) + if n == 0 { + return true + } + if good, known := goodOS[l[n-1]]; known { + return good + } + if good, known := goodArch[l[n-1]]; known { + if !good || n < 2 { + return false + } + good, known = goodOS[l[n-2]] + return good || !known + } + return true +} + +var goodOS = make(map[string]bool) +var goodArch = make(map[string]bool) + +func init() { + goodOS = make(map[string]bool) + goodArch = make(map[string]bool) + for _, v := range strings.Fields(goosList) { + goodOS[v] = v == runtime.GOOS + } + for _, v := range strings.Fields(goarchList) { + goodArch[v] = v == runtime.GOARCH + } +} diff --git a/src/pkg/go/build/path.go b/src/pkg/go/build/path.go new file mode 100644 index 000000000..8ad39fb0f --- /dev/null +++ b/src/pkg/go/build/path.go @@ -0,0 +1,163 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "fmt" + "log" + "os" + "path/filepath" + "runtime" + "strings" +) + +// Path is a validated list of Trees derived from $GOPATH at init. +var Path []*Tree + +// Tree describes a Go source tree, either $GOROOT or one from $GOPATH. +type Tree struct { + Path string + Goroot bool +} + +func newTree(p string) (*Tree, os.Error) { + if !filepath.IsAbs(p) { + return nil, os.NewError("must be absolute") + } + ep, err := filepath.EvalSymlinks(p) + if err != nil { + return nil, err + } + return &Tree{Path: ep}, nil +} + +// SrcDir returns the tree's package source directory. +func (t *Tree) SrcDir() string { + if t.Goroot { + return filepath.Join(t.Path, "src", "pkg") + } + return filepath.Join(t.Path, "src") +} + +// PkgDir returns the tree's package object directory. +func (t *Tree) PkgDir() string { + goos, goarch := runtime.GOOS, runtime.GOARCH + if e := os.Getenv("GOOS"); e != "" { + goos = e + } + if e := os.Getenv("GOARCH"); e != "" { + goarch = e + } + return filepath.Join(t.Path, "pkg", goos+"_"+goarch) +} + +// BinDir returns the tree's binary executable directory. +func (t *Tree) BinDir() string { + return filepath.Join(t.Path, "bin") +} + +// HasSrc returns whether the given package's +// source can be found inside this Tree. +func (t *Tree) HasSrc(pkg string) bool { + fi, err := os.Stat(filepath.Join(t.SrcDir(), pkg)) + if err != nil { + return false + } + return fi.IsDirectory() +} + +// HasPkg returns whether the given package's +// object file can be found inside this Tree. +func (t *Tree) HasPkg(pkg string) bool { + fi, err := os.Stat(filepath.Join(t.PkgDir(), pkg+".a")) + if err != nil { + return false + } + return fi.IsRegular() + // TODO(adg): check object version is consistent +} + +var ErrNotFound = os.NewError("package could not be found locally") + +// FindTree takes an import or filesystem path and returns the +// tree where the package source should be and the package import path. +func FindTree(path string) (tree *Tree, pkg string, err os.Error) { + if isLocalPath(path) { + if path, err = filepath.Abs(path); err != nil { + return + } + for _, t := range Path { + tpath := t.SrcDir() + string(filepath.Separator) + if !strings.HasPrefix(path, tpath) { + continue + } + tree = t + pkg = path[len(tpath):] + return + } + err = fmt.Errorf("path %q not inside a GOPATH", path) + return + } + tree = defaultTree + pkg = path + for _, t := range Path { + if t.HasSrc(pkg) { + tree = t + return + } + } + err = ErrNotFound + return +} + +// isLocalPath returns whether the given path is local (/foo ./foo ../foo . ..) +func isLocalPath(s string) bool { + const sep = string(filepath.Separator) + return strings.HasPrefix(s, sep) || strings.HasPrefix(s, "."+sep) || strings.HasPrefix(s, ".."+sep) || s == "." || s == ".." +} + +var ( + // argument lists used by the build's gc and ld methods + gcImportArgs []string + ldImportArgs []string + + // default tree for remote packages + defaultTree *Tree +) + +// set up Path: parse and validate GOROOT and GOPATH variables +func init() { + root := runtime.GOROOT() + p, err := newTree(root) + if err != nil { + log.Fatalf("Invalid GOROOT %q: %v", root, err) + } + p.Goroot = true + Path = []*Tree{p} + + for _, p := range filepath.SplitList(os.Getenv("GOPATH")) { + if p == "" { + continue + } + t, err := newTree(p) + if err != nil { + log.Printf("Invalid GOPATH %q: %v", p, err) + continue + } + Path = append(Path, t) + gcImportArgs = append(gcImportArgs, "-I", t.PkgDir()) + ldImportArgs = append(ldImportArgs, "-L", t.PkgDir()) + + // select first GOPATH entry as default + if defaultTree == nil { + defaultTree = t + } + } + + // use GOROOT if no valid GOPATH specified + if defaultTree == nil { + defaultTree = Path[0] + } +} diff --git a/src/pkg/go/build/syslist_test.go b/src/pkg/go/build/syslist_test.go new file mode 100644 index 000000000..eb0e5dcb6 --- /dev/null +++ b/src/pkg/go/build/syslist_test.go @@ -0,0 +1,62 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "runtime" + "testing" +) + +var ( + thisOS = runtime.GOOS + thisArch = runtime.GOARCH + otherOS = anotherOS() + otherArch = anotherArch() +) + +func anotherOS() string { + if thisOS != "darwin" { + return "darwin" + } + return "linux" +} + +func anotherArch() string { + if thisArch != "amd64" { + return "amd64" + } + return "386" +} + +type GoodFileTest struct { + name string + result bool +} + +var tests = []GoodFileTest{ + {"file.go", true}, + {"file.c", true}, + {"file_foo.go", true}, + {"file_" + thisArch + ".go", true}, + {"file_" + otherArch + ".go", false}, + {"file_" + thisOS + ".go", true}, + {"file_" + otherOS + ".go", false}, + {"file_" + thisOS + "_" + thisArch + ".go", true}, + {"file_" + otherOS + "_" + thisArch + ".go", false}, + {"file_" + thisOS + "_" + otherArch + ".go", false}, + {"file_" + otherOS + "_" + otherArch + ".go", false}, + {"file_foo_" + thisArch + ".go", true}, + {"file_foo_" + otherArch + ".go", false}, + {"file_" + thisOS + ".c", true}, + {"file_" + otherOS + ".c", false}, +} + +func TestGoodOSArch(t *testing.T) { + for _, test := range tests { + if goodOSArch(test.name) != test.result { + t.Fatalf("goodOSArch(%q) != %v", test.name, test.result) + } + } +} diff --git a/src/pkg/go/doc/doc.go b/src/pkg/go/doc/doc.go index 29d205d39..a7a7e0a32 100644 --- a/src/pkg/go/doc/doc.go +++ b/src/pkg/go/doc/doc.go @@ -572,6 +572,20 @@ func (doc *docReader) newDoc(importpath string, filenames []string) *PackageDoc type Filter func(string) bool +func matchFields(fields *ast.FieldList, f Filter) bool { + if fields != nil { + for _, field := range fields.List { + for _, name := range field.Names { + if f(name.Name) { + return true + } + } + } + } + return false +} + + func matchDecl(d *ast.GenDecl, f Filter) bool { for _, d := range d.Specs { switch v := d.(type) { @@ -585,6 +599,16 @@ func matchDecl(d *ast.GenDecl, f Filter) bool { if f(v.Name.Name) { return true } + switch t := v.Type.(type) { + case *ast.StructType: + if matchFields(t.Fields, f) { + return true + } + case *ast.InterfaceType: + if matchFields(t.Methods, f) { + return true + } + } } } return false diff --git a/src/pkg/go/parser/parser.go b/src/pkg/go/parser/parser.go index 5c57e41d1..586ee3a9a 100644 --- a/src/pkg/go/parser/parser.go +++ b/src/pkg/go/parser/parser.go @@ -54,7 +54,7 @@ type parser struct { // Non-syntactic parser control exprLev int // < 0: in control clause, >= 0: in expression - // Ordinary identifer scopes + // Ordinary identifier scopes pkgScope *ast.Scope // pkgScope.Outer == nil topScope *ast.Scope // top-most scope; may be pkgScope unresolved []*ast.Ident // unresolved identifiers @@ -131,14 +131,16 @@ func (p *parser) closeLabelScope() { } -func (p *parser) declare(decl interface{}, scope *ast.Scope, kind ast.ObjKind, idents ...*ast.Ident) { +func (p *parser) declare(decl, data interface{}, scope *ast.Scope, kind ast.ObjKind, idents ...*ast.Ident) { for _, ident := range idents { assert(ident.Obj == nil, "identifier already declared or resolved") + obj := ast.NewObj(kind, ident.Name) + // remember the corresponding declaration for redeclaration + // errors and global variable resolution/typechecking phase + obj.Decl = decl + obj.Data = data + ident.Obj = obj if ident.Name != "_" { - obj := ast.NewObj(kind, ident.Name) - // remember the corresponding declaration for redeclaration - // errors and global variable resolution/typechecking phase - obj.Decl = decl if alt := scope.Insert(obj); alt != nil && p.mode&DeclarationErrors != 0 { prevDecl := "" if pos := alt.Pos(); pos.IsValid() { @@ -146,7 +148,6 @@ func (p *parser) declare(decl interface{}, scope *ast.Scope, kind ast.ObjKind, i } p.error(ident.Pos(), fmt.Sprintf("%s redeclared in this block%s", ident.Name, prevDecl)) } - ident.Obj = obj } } } @@ -159,17 +160,17 @@ func (p *parser) shortVarDecl(idents []*ast.Ident) { n := 0 // number of new variables for _, ident := range idents { assert(ident.Obj == nil, "identifier already declared or resolved") + obj := ast.NewObj(ast.Var, ident.Name) + // short var declarations cannot have redeclaration errors + // and are not global => no need to remember the respective + // declaration + ident.Obj = obj if ident.Name != "_" { - obj := ast.NewObj(ast.Var, ident.Name) - // short var declarations cannot have redeclaration errors - // and are not global => no need to remember the respective - // declaration - alt := p.topScope.Insert(obj) - if alt == nil { + if alt := p.topScope.Insert(obj); alt != nil { + ident.Obj = alt // redeclaration + } else { n++ // new declaration - alt = obj } - ident.Obj = alt } } if n == 0 && p.mode&DeclarationErrors != 0 { @@ -596,7 +597,7 @@ func (p *parser) parseFieldDecl(scope *ast.Scope) *ast.Field { p.expectSemi() // call before accessing p.linecomment field := &ast.Field{doc, idents, typ, tag, p.lineComment} - p.declare(field, scope, ast.Var, idents...) + p.declare(field, nil, scope, ast.Var, idents...) return field } @@ -707,7 +708,7 @@ func (p *parser) parseParameterList(scope *ast.Scope, ellipsisOk bool) (params [ params = append(params, field) // Go spec: The scope of an identifier denoting a function // parameter or result variable is the function body. - p.declare(field, scope, ast.Var, idents...) + p.declare(field, nil, scope, ast.Var, idents...) if p.tok == token.COMMA { p.next() } @@ -719,7 +720,7 @@ func (p *parser) parseParameterList(scope *ast.Scope, ellipsisOk bool) (params [ params = append(params, field) // Go spec: The scope of an identifier denoting a function // parameter or result variable is the function body. - p.declare(field, scope, ast.Var, idents...) + p.declare(field, nil, scope, ast.Var, idents...) if p.tok != token.COMMA { break } @@ -818,11 +819,12 @@ func (p *parser) parseMethodSpec(scope *ast.Scope) *ast.Field { } else { // embedded interface typ = x + p.resolve(typ) } p.expectSemi() // call before accessing p.linecomment spec := &ast.Field{doc, idents, typ, nil, p.lineComment} - p.declare(spec, scope, ast.Fun, idents...) + p.declare(spec, nil, scope, ast.Fun, idents...) return spec } @@ -1476,7 +1478,7 @@ func (p *parser) parseSimpleStmt(labelOk bool) ast.Stmt { // in which it is declared and excludes the body of any nested // function. stmt := &ast.LabeledStmt{label, colon, p.parseStmt()} - p.declare(stmt, p.labelScope, ast.Lbl, label) + p.declare(stmt, nil, p.labelScope, ast.Lbl, label) return stmt } p.error(x[0].Pos(), "illegal label declaration") @@ -1780,10 +1782,6 @@ func (p *parser) parseCommClause() *ast.CommClause { rhs = lhs[0] lhs = nil // there is no lhs } - if x, isUnary := rhs.(*ast.UnaryExpr); !isUnary || x.Op != token.ARROW { - p.errorExpected(rhs.Pos(), "send or receive operation") - rhs = &ast.BadExpr{rhs.Pos(), rhs.End()} - } if lhs != nil { comm = &ast.AssignStmt{lhs, pos, tok, []ast.Expr{rhs}} } else { @@ -2004,7 +2002,7 @@ func parseConstSpec(p *parser, doc *ast.CommentGroup, iota int) ast.Spec { // the end of the innermost containing block. // (Global identifiers are resolved in a separate phase after parsing.) spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment} - p.declare(spec, p.topScope, ast.Con, idents...) + p.declare(spec, iota, p.topScope, ast.Con, idents...) return spec } @@ -2022,7 +2020,7 @@ func parseTypeSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec { // containing block. // (Global identifiers are resolved in a separate phase after parsing.) spec := &ast.TypeSpec{doc, ident, nil, nil} - p.declare(spec, p.topScope, ast.Typ, ident) + p.declare(spec, nil, p.topScope, ast.Typ, ident) spec.Type = p.parseType() p.expectSemi() // call before accessing p.linecomment @@ -2051,7 +2049,7 @@ func parseVarSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec { // the end of the innermost containing block. // (Global identifiers are resolved in a separate phase after parsing.) spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment} - p.declare(spec, p.topScope, ast.Var, idents...) + p.declare(spec, nil, p.topScope, ast.Var, idents...) return spec } @@ -2143,7 +2141,7 @@ func (p *parser) parseFuncDecl() *ast.FuncDecl { // init() functions cannot be referred to and there may // be more than one - don't put them in the pkgScope if ident.Name != "init" { - p.declare(decl, p.pkgScope, ast.Fun, ident) + p.declare(decl, nil, p.pkgScope, ast.Fun, ident) } } diff --git a/src/pkg/go/parser/parser_test.go b/src/pkg/go/parser/parser_test.go index 2f1ee6bfc..5b52f51d4 100644 --- a/src/pkg/go/parser/parser_test.go +++ b/src/pkg/go/parser/parser_test.go @@ -51,6 +51,7 @@ var validPrograms = []interface{}{ `package p; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`, `package p; var a = T{{1, 2}, {3, 4}}`, `package p; func f() { select { case <- c: case c <- d: case c <- <- d: case <-c <- d: } };`, + `package p; func f() { select { case x := (<-c): } };`, `package p; func f() { if ; true {} };`, `package p; func f() { switch ; {} };`, } diff --git a/src/pkg/go/printer/nodes.go b/src/pkg/go/printer/nodes.go index 86c327930..0fca8a161 100644 --- a/src/pkg/go/printer/nodes.go +++ b/src/pkg/go/printer/nodes.go @@ -33,7 +33,7 @@ import ( // line break was printed; returns false otherwise. // // TODO(gri): linebreak may add too many lines if the next statement at "line" -// is preceeded by comments because the computation of n assumes +// is preceded by comments because the computation of n assumes // the current position before the comment and the target position // after the comment. Thus, after interspersing such comments, the // space taken up by them is not considered to reduce the number of @@ -215,12 +215,13 @@ func (p *printer) exprList(prev0 token.Pos, list []ast.Expr, depth int, mode exp } if i > 0 { - if mode&commaSep != 0 { + switch { + case mode&commaSep != 0: p.print(token.COMMA) - } - if mode&periodSep != 0 { + case mode&periodSep != 0: p.print(token.PERIOD) } + needsBlank := mode&periodSep == 0 // period-separated list elements don't need a blank if prevLine < line && prevLine > 0 && line > 0 { // lines are broken using newlines so comments remain aligned // unless forceFF is set or there are multiple expressions on @@ -229,11 +230,12 @@ func (p *printer) exprList(prev0 token.Pos, list []ast.Expr, depth int, mode exp ws = ignore *multiLine = true prevBreak = i + needsBlank = false // we got a line break instead } - } else if mode&periodSep == 0 { + } + if needsBlank { p.print(blank) } - // period-separated list elements don't need a blank } if isPair && size > 0 && len(list) > 1 { @@ -438,8 +440,8 @@ func (p *printer) fieldList(fields *ast.FieldList, isStruct, isIncomplete bool) if len(list) > 0 { p.print(formfeed) } - p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't loose the last line comment - p.setLineComment("// contains unexported fields") + p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't lose the last line comment + p.setLineComment("// contains filtered or unexported fields") } } else { // interface @@ -465,8 +467,8 @@ func (p *printer) fieldList(fields *ast.FieldList, isStruct, isIncomplete bool) if len(list) > 0 { p.print(formfeed) } - p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't loose the last line comment - p.setLineComment("// contains unexported methods") + p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't lose the last line comment + p.setLineComment("// contains filtered or unexported methods") } } @@ -1189,6 +1191,97 @@ func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, multiLine *bool) { // ---------------------------------------------------------------------------- // Declarations +// The keepTypeColumn function determines if the type column of a series of +// consecutive const or var declarations must be kept, or if initialization +// values (V) can be placed in the type column (T) instead. The i'th entry +// in the result slice is true if the type column in spec[i] must be kept. +// +// For example, the declaration: +// +// const ( +// foobar int = 42 // comment +// x = 7 // comment +// foo +// bar = 991 +// ) +// +// leads to the type/values matrix below. A run of value columns (V) can +// be moved into the type column if there is no type for any of the values +// in that column (we only move entire columns so that they align properly). +// +// matrix formatted result +// matrix +// T V -> T V -> true there is a T and so the type +// - V - V true column must be kept +// - - - - false +// - V V - false V is moved into T column +// +func keepTypeColumn(specs []ast.Spec) []bool { + m := make([]bool, len(specs)) + + populate := func(i, j int, keepType bool) { + if keepType { + for ; i < j; i++ { + m[i] = true + } + } + } + + i0 := -1 // if i0 >= 0 we are in a run and i0 is the start of the run + var keepType bool + for i, s := range specs { + t := s.(*ast.ValueSpec) + if t.Values != nil { + if i0 < 0 { + // start of a run of ValueSpecs with non-nil Values + i0 = i + keepType = false + } + } else { + if i0 >= 0 { + // end of a run + populate(i0, i, keepType) + i0 = -1 + } + } + if t.Type != nil { + keepType = true + } + } + if i0 >= 0 { + // end of a run + populate(i0, len(specs), keepType) + } + + return m +} + + +func (p *printer) valueSpec(s *ast.ValueSpec, keepType, doIndent bool, multiLine *bool) { + p.setComment(s.Doc) + p.identList(s.Names, doIndent, multiLine) // always present + extraTabs := 3 + if s.Type != nil || keepType { + p.print(vtab) + extraTabs-- + } + if s.Type != nil { + p.expr(s.Type, multiLine) + } + if s.Values != nil { + p.print(vtab, token.ASSIGN) + p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos) + extraTabs-- + } + if s.Comment != nil { + for ; extraTabs > 0; extraTabs-- { + p.print(vtab) + } + p.setComment(s.Comment) + } +} + + // The parameter n is the number of specs in the group. If doIndent is set, // multi-line identifier lists in the spec are indented when the first // linebreak is encountered. @@ -1206,38 +1299,20 @@ func (p *printer) spec(spec ast.Spec, n int, doIndent bool, multiLine *bool) { p.setComment(s.Comment) case *ast.ValueSpec: + if n != 1 { + p.internalError("expected n = 1; got", n) + } p.setComment(s.Doc) p.identList(s.Names, doIndent, multiLine) // always present - if n == 1 { - if s.Type != nil { - p.print(blank) - p.expr(s.Type, multiLine) - } - if s.Values != nil { - p.print(blank, token.ASSIGN) - p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos) - } - p.setComment(s.Comment) - - } else { - extraTabs := 3 - if s.Type != nil { - p.print(vtab) - p.expr(s.Type, multiLine) - extraTabs-- - } - if s.Values != nil { - p.print(vtab, token.ASSIGN) - p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos) - extraTabs-- - } - if s.Comment != nil { - for ; extraTabs > 0; extraTabs-- { - p.print(vtab) - } - p.setComment(s.Comment) - } + if s.Type != nil { + p.print(blank) + p.expr(s.Type, multiLine) + } + if s.Values != nil { + p.print(blank, token.ASSIGN) + p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos) } + p.setComment(s.Comment) case *ast.TypeSpec: p.setComment(s.Doc) @@ -1264,15 +1339,29 @@ func (p *printer) genDecl(d *ast.GenDecl, multiLine *bool) { if d.Lparen.IsValid() { // group of parenthesized declarations p.print(d.Lparen, token.LPAREN) - if len(d.Specs) > 0 { + if n := len(d.Specs); n > 0 { p.print(indent, formfeed) - var ml bool - for i, s := range d.Specs { - if i > 0 { - p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, ml) + if n > 1 && (d.Tok == token.CONST || d.Tok == token.VAR) { + // two or more grouped const/var declarations: + // determine if the type column must be kept + keepType := keepTypeColumn(d.Specs) + var ml bool + for i, s := range d.Specs { + if i > 0 { + p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, ml) + } + ml = false + p.valueSpec(s.(*ast.ValueSpec), keepType[i], false, &ml) + } + } else { + var ml bool + for i, s := range d.Specs { + if i > 0 { + p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, ml) + } + ml = false + p.spec(s, n, false, &ml) } - ml = false - p.spec(s, len(d.Specs), false, &ml) } p.print(unindent, formfeed) *multiLine = true @@ -1303,7 +1392,7 @@ func (p *printer) nodeSize(n ast.Node, maxSize int) (size int) { size = maxSize + 1 // assume n doesn't fit p.nodeSizes[n] = size - // nodeSize computation must be indendent of particular + // nodeSize computation must be independent of particular // style so that we always get the same decision; print // in RawFormat cfg := Config{Mode: RawFormat} diff --git a/src/pkg/go/printer/performance_test.go b/src/pkg/go/printer/performance_test.go new file mode 100644 index 000000000..31de0b7ad --- /dev/null +++ b/src/pkg/go/printer/performance_test.go @@ -0,0 +1,62 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements a simple printer performance benchmark: +// gotest -bench=BenchmarkPrint + +package printer + +import ( + "bytes" + "go/ast" + "go/parser" + "io" + "io/ioutil" + "log" + "testing" +) + + +var testfile *ast.File + + +func testprint(out io.Writer, file *ast.File) { + if _, err := (&Config{TabIndent | UseSpaces, 8}).Fprint(out, fset, file); err != nil { + log.Fatalf("print error: %s", err) + } +} + + +// cannot initialize in init because (printer) Fprint launches goroutines. +func initialize() { + const filename = "testdata/parser.go" + + src, err := ioutil.ReadFile(filename) + if err != nil { + log.Fatalf("%s", err) + } + + file, err := parser.ParseFile(fset, filename, src, parser.ParseComments) + if err != nil { + log.Fatalf("%s", err) + } + + var buf bytes.Buffer + testprint(&buf, file) + if !bytes.Equal(buf.Bytes(), src) { + log.Fatalf("print error: %s not idempotent", filename) + } + + testfile = file +} + + +func BenchmarkPrint(b *testing.B) { + if testfile == nil { + initialize() + } + for i := 0; i < b.N; i++ { + testprint(ioutil.Discard, testfile) + } +} diff --git a/src/pkg/go/printer/printer.go b/src/pkg/go/printer/printer.go index 01ebf783c..40b15dd79 100644 --- a/src/pkg/go/printer/printer.go +++ b/src/pkg/go/printer/printer.go @@ -589,7 +589,7 @@ func (p *printer) writeCommentSuffix(needsLinebreak bool) (droppedFF bool) { // ignore trailing whitespace p.wsbuf[i] = ignore case indent, unindent: - // don't loose indentation information + // don't lose indentation information case newline, formfeed: // if we need a line break, keep exactly one // but remember if we dropped any formfeeds diff --git a/src/pkg/go/printer/testdata/comments.golden b/src/pkg/go/printer/testdata/comments.golden index a86d66174..b177c3571 100644 --- a/src/pkg/go/printer/testdata/comments.golden +++ b/src/pkg/go/printer/testdata/comments.golden @@ -22,7 +22,7 @@ const ( _ = iota + 10 _ // comments - _ = 10 // comment + _ = 10 // comment _ T = 20 // comment ) @@ -38,9 +38,9 @@ const ( _ // comment _ // comment _ = iota + 10 - _ // comment - _ = 10 - _ = 20 // comment + _ // comment + _ = 10 + _ = 20 // comment _ T = 0 // comment ) @@ -436,7 +436,7 @@ func _() { // Comments immediately adjacent to punctuation (for which the go/printer -// may obly have estimated position information) must remain after the punctuation. +// may only have estimated position information) must remain after the punctuation. func _() { _ = T{ 1, // comment after comma diff --git a/src/pkg/go/printer/testdata/comments.input b/src/pkg/go/printer/testdata/comments.input index 14cd4cf7a..2a9a86b68 100644 --- a/src/pkg/go/printer/testdata/comments.input +++ b/src/pkg/go/printer/testdata/comments.input @@ -434,7 +434,7 @@ func _() { // Comments immediately adjacent to punctuation (for which the go/printer -// may obly have estimated position information) must remain after the punctuation. +// may only have estimated position information) must remain after the punctuation. func _() { _ = T{ 1, // comment after comma diff --git a/src/pkg/go/printer/testdata/comments.x b/src/pkg/go/printer/testdata/comments.x index 4d7a928ae..30a182f49 100644 --- a/src/pkg/go/printer/testdata/comments.x +++ b/src/pkg/go/printer/testdata/comments.x @@ -8,7 +8,7 @@ type SZ struct{} // The S0 struct; no field is exported. type S0 struct { - // contains unexported fields + // contains filtered or unexported fields } // The S1 struct; some fields are not exported. @@ -16,7 +16,7 @@ type S1 struct { S0 A, B, C float // 3 exported fields D int // 2 unexported fields - // contains unexported fields + // contains filtered or unexported fields } // The S2 struct; all fields are exported. @@ -30,14 +30,14 @@ type SZ interface{} // The I0 interface; no method is exported. type I0 interface { - // contains unexported methods + // contains filtered or unexported methods } // The I1 interface; some methods are not exported. type I1 interface { I0 F(x float) float // exported methods - // contains unexported methods + // contains filtered or unexported methods } // The I2 interface; all methods are exported. @@ -53,5 +53,5 @@ type S3 struct { F1 int // line comment for F1 // lead comment for F2 F2 int // line comment for F2 - // contains unexported fields + // contains filtered or unexported fields } diff --git a/src/pkg/go/printer/testdata/declarations.golden b/src/pkg/go/printer/testdata/declarations.golden index c1b255842..fac72f651 100644 --- a/src/pkg/go/printer/testdata/declarations.golden +++ b/src/pkg/go/printer/testdata/declarations.golden @@ -160,7 +160,6 @@ bar` func _() { - // the following decls need a semicolon at the end type _ int type _ *int type _ []int @@ -175,7 +174,6 @@ func _() { var _ chan int var _ func() int - // the following decls don't need a semicolon at the end type _ struct{} type _ *struct{} type _ []struct{} @@ -331,11 +329,11 @@ func _() { ) // some entries have a type const ( - xxxxxx = 1 - x = 2 - xxx = 3 + xxxxxx = 1 + x = 2 + xxx = 3 yyyyyyyy float = iota - yyyy = "bar" + yyyy = "bar" yyy yy = 2 ) @@ -365,7 +363,7 @@ func _() { xxx string yyyyyyyy int = 1234 y float = 3.14 - yyyy = "bar" + yyyy = "bar" yyy string = "foo" ) // mixed entries - all comments should be aligned @@ -373,7 +371,7 @@ func _() { a, b, c int x = 10 d int // comment - y = 20 // comment + y = 20 // comment f, ff, fff, ffff int = 0, 1, 2, 3 // comment ) // respect original line breaks @@ -401,6 +399,33 @@ func _() { ) } +// alignment of "=" in consecutive lines (extended example from issue 1414) +const ( + umax uint = ^uint(0) // maximum value for a uint + bpu = 1 << (5 + umax>>63) // bits per uint + foo + bar = -1 +) + +// typical enum +const ( + a MyType = iota + abcd + b + c + def +) + +// excerpt from godoc.go +var ( + goroot = flag.String("goroot", runtime.GOROOT(), "Go root directory") + testDir = flag.String("testdir", "", "Go root subdirectory - for testing only (faster startups)") + pkgPath = flag.String("path", "", "additional package directories (colon-separated)") + filter = flag.String("filter", "", "filter file containing permitted package directory paths") + filterMin = flag.Int("filter_minutes", 0, "filter file update interval in minutes; disabled if <= 0") + filterDelay delayTime // actual filter update interval in minutes; usually filterDelay == filterMin, but filterDelay may back off exponentially +) + // formatting of structs type _ struct{} diff --git a/src/pkg/go/printer/testdata/declarations.input b/src/pkg/go/printer/testdata/declarations.input index c8b37e12b..c6134096b 100644 --- a/src/pkg/go/printer/testdata/declarations.input +++ b/src/pkg/go/printer/testdata/declarations.input @@ -159,7 +159,6 @@ bar` func _() { - // the following decls need a semicolon at the end type _ int type _ *int type _ []int @@ -174,7 +173,6 @@ func _() { var _ chan int var _ func() int - // the following decls don't need a semicolon at the end type _ struct{} type _ *struct{} type _ []struct{} @@ -400,6 +398,33 @@ func _() { ) } +// alignment of "=" in consecutive lines (extended example from issue 1414) +const ( + umax uint = ^uint(0) // maximum value for a uint + bpu = 1 << (5 + umax>>63) // bits per uint + foo + bar = -1 +) + +// typical enum +const ( + a MyType = iota + abcd + b + c + def +) + +// excerpt from godoc.go +var ( + goroot = flag.String("goroot", runtime.GOROOT(), "Go root directory") + testDir = flag.String("testdir", "", "Go root subdirectory - for testing only (faster startups)") + pkgPath = flag.String("path", "", "additional package directories (colon-separated)") + filter = flag.String("filter", "", "filter file containing permitted package directory paths") + filterMin = flag.Int("filter_minutes", 0, "filter file update interval in minutes; disabled if <= 0") + filterDelay delayTime // actual filter update interval in minutes; usually filterDelay == filterMin, but filterDelay may back off exponentially +) + // formatting of structs type _ struct{} diff --git a/src/pkg/go/printer/testdata/expressions.golden b/src/pkg/go/printer/testdata/expressions.golden index c1a7e970b..a5e2fdc3b 100644 --- a/src/pkg/go/printer/testdata/expressions.golden +++ b/src/pkg/go/printer/testdata/expressions.golden @@ -94,30 +94,49 @@ func _() { _ = under_bar - 1 _ = Open(dpath+"/file", O_WRONLY|O_CREAT, 0666) _ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx) -} - -func _() { + // the parser does not restrict expressions that may appear as statements + true + 42 + "foo" + x + (x) a + b a + b + c - a + b*c a + (b * c) - (a + b) * c - a + (b * c * d) - a + (b*c + d) + a + (b / c) + 1 + a + a + 1 + s[a] + x << 1 + (s[0] << 1) & 0xf + "foo" + s + x == y + x < y || z > 42 +} + + +func _() { + _ = a + b + _ = a + b + c + _ = a + b*c + _ = a + (b * c) + _ = (a + b) * c + _ = a + (b * c * d) + _ = a + (b*c + d) - 1 << x - -1 << x - 1<<x - 1 - -1<<x - 1 + _ = 1 << x + _ = -1 << x + _ = 1<<x - 1 + _ = -1<<x - 1 - f(a + b) - f(a + b + c) - f(a + b*c) - f(a + (b * c)) - f(1<<x-1, 1<<x-2) + _ = f(a + b) + _ = f(a + b + c) + _ = f(a + b*c) + _ = f(a + (b * c)) + _ = f(1<<x-1, 1<<x-2) - 1<<d.logWindowSize - 1 + _ = 1<<d.logWindowSize - 1 buf = make(x, 2*cap(b.buf)+n) @@ -131,7 +150,7 @@ func _() { signed += ' ' * 8 tw.octal(header[148:155], chksum) - x > 0 && i >= 0 + _ = x > 0 && i >= 0 x1, x0 := x>>w2, x&m2 z0 = t1<<w2 + t0 @@ -141,31 +160,31 @@ func _() { x1 = (x1 << z) | (x0 >> (uint(w) - z)) x1 = x1<<z | x0>>(uint(w)-z) - buf[0 : len(buf)+1] - buf[0 : n+1] + _ = buf[0 : len(buf)+1] + _ = buf[0 : n+1] a, b = b, a a = b + c a = b*c + d - a*b + c - a - b - c - a - (b - c) - a - b*c - a - (b * c) - a * b / c - a / *b - x[a|^b] - x[a / *b] - a & ^b - a + +b - a - -b - x[a*-b] - x[a + +b] - x ^ y ^ z - b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF] - len(longVariableName) * 2 - - token(matchType + xlength<<lengthShift + xoffset) + _ = a*b + c + _ = a - b - c + _ = a - (b - c) + _ = a - b*c + _ = a - (b * c) + _ = a * b / c + _ = a / *b + _ = x[a|^b] + _ = x[a / *b] + _ = a & ^b + _ = a + +b + _ = a - -b + _ = x[a*-b] + _ = x[a + +b] + _ = x ^ y ^ z + _ = b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF] + _ = len(longVariableName) * 2 + + _ = token(matchType + xlength<<lengthShift + xoffset) } @@ -625,7 +644,7 @@ func _() { func f() { // os.Open parameters should remain on two lines if writer, err = os.Open(outfile, s.O_WRONLY|os.O_CREATE| - os.O_TRUNC,0666); err != nil { + os.O_TRUNC, 0666); err != nil { log.Fatal(err) } } diff --git a/src/pkg/go/printer/testdata/expressions.input b/src/pkg/go/printer/testdata/expressions.input index b87381198..d11314983 100644 --- a/src/pkg/go/printer/testdata/expressions.input +++ b/src/pkg/go/printer/testdata/expressions.input @@ -94,30 +94,49 @@ func _() { _ = under_bar-1 _ = Open(dpath + "/file", O_WRONLY | O_CREAT, 0666) _ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx) -} - -func _() { + // the parser does not restrict expressions that may appear as statements + true + 42 + "foo" + x + (x) a+b a+b+c - a+b*c a+(b*c) - (a+b)*c - a+(b*c*d) - a+(b*c+d) + a+(b/c) + 1+a + a+1 + s[a] + x<<1 + (s[0]<<1)&0xf + "foo"+s + x == y + x < y || z > 42 +} - 1<<x - -1<<x - 1<<x-1 - -1<<x-1 - f(a+b) - f(a+b+c) - f(a+b*c) - f(a+(b*c)) - f(1<<x-1, 1<<x-2) +func _() { + _ = a+b + _ = a+b+c + _ = a+b*c + _ = a+(b*c) + _ = (a+b)*c + _ = a+(b*c*d) + _ = a+(b*c+d) + + _ = 1<<x + _ = -1<<x + _ = 1<<x-1 + _ = -1<<x-1 - 1<<d.logWindowSize-1 + _ = f(a+b) + _ = f(a+b+c) + _ = f(a+b*c) + _ = f(a+(b*c)) + _ = f(1<<x-1, 1<<x-2) + + _ = 1<<d.logWindowSize-1 buf = make(x, 2*cap(b.buf) + n) @@ -131,7 +150,7 @@ func _() { signed += ' '*8 tw.octal(header[148:155], chksum) - x > 0 && i >= 0 + _ = x > 0 && i >= 0 x1, x0 := x>>w2, x&m2 z0 = t1<<w2+t0 @@ -141,31 +160,31 @@ func _() { x1 = (x1<<z)|(x0>>(uint(w)-z)) x1 = x1<<z | x0>>(uint(w)-z) - buf[0:len(buf)+1] - buf[0:n+1] + _ = buf[0:len(buf)+1] + _ = buf[0:n+1] a,b = b,a a = b+c a = b*c+d - a*b+c - a-b-c - a-(b-c) - a-b*c - a-(b*c) - a*b/c - a/ *b - x[a|^b] - x[a/ *b] - a& ^b - a+ +b - a- -b - x[a*-b] - x[a+ +b] - x^y^z - b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF] - len(longVariableName)*2 - - token(matchType + xlength<<lengthShift + xoffset) + _ = a*b+c + _ = a-b-c + _ = a-(b-c) + _ = a-b*c + _ = a-(b*c) + _ = a*b/c + _ = a/ *b + _ = x[a|^b] + _ = x[a/ *b] + _ = a& ^b + _ = a+ +b + _ = a- -b + _ = x[a*-b] + _ = x[a+ +b] + _ = x^y^z + _ = b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF] + _ = len(longVariableName)*2 + + _ = token(matchType + xlength<<lengthShift + xoffset) } diff --git a/src/pkg/go/printer/testdata/expressions.raw b/src/pkg/go/printer/testdata/expressions.raw index 735cd943e..308d9edff 100644 --- a/src/pkg/go/printer/testdata/expressions.raw +++ b/src/pkg/go/printer/testdata/expressions.raw @@ -94,30 +94,49 @@ func _() { _ = under_bar - 1 _ = Open(dpath+"/file", O_WRONLY|O_CREAT, 0666) _ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx) -} - -func _() { + // the parser does not restrict expressions that may appear as statements + true + 42 + "foo" + x + (x) a + b a + b + c - a + b*c a + (b * c) - (a + b) * c - a + (b * c * d) - a + (b*c + d) + a + (b / c) + 1 + a + a + 1 + s[a] + x << 1 + (s[0] << 1) & 0xf + "foo" + s + x == y + x < y || z > 42 +} + + +func _() { + _ = a + b + _ = a + b + c + _ = a + b*c + _ = a + (b * c) + _ = (a + b) * c + _ = a + (b * c * d) + _ = a + (b*c + d) - 1 << x - -1 << x - 1<<x - 1 - -1<<x - 1 + _ = 1 << x + _ = -1 << x + _ = 1<<x - 1 + _ = -1<<x - 1 - f(a + b) - f(a + b + c) - f(a + b*c) - f(a + (b * c)) - f(1<<x-1, 1<<x-2) + _ = f(a + b) + _ = f(a + b + c) + _ = f(a + b*c) + _ = f(a + (b * c)) + _ = f(1<<x-1, 1<<x-2) - 1<<d.logWindowSize - 1 + _ = 1<<d.logWindowSize - 1 buf = make(x, 2*cap(b.buf)+n) @@ -131,7 +150,7 @@ func _() { signed += ' ' * 8 tw.octal(header[148:155], chksum) - x > 0 && i >= 0 + _ = x > 0 && i >= 0 x1, x0 := x>>w2, x&m2 z0 = t1<<w2 + t0 @@ -141,31 +160,31 @@ func _() { x1 = (x1 << z) | (x0 >> (uint(w) - z)) x1 = x1<<z | x0>>(uint(w)-z) - buf[0 : len(buf)+1] - buf[0 : n+1] + _ = buf[0 : len(buf)+1] + _ = buf[0 : n+1] a, b = b, a a = b + c a = b*c + d - a*b + c - a - b - c - a - (b - c) - a - b*c - a - (b * c) - a * b / c - a / *b - x[a|^b] - x[a / *b] - a & ^b - a + +b - a - -b - x[a*-b] - x[a + +b] - x ^ y ^ z - b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF] - len(longVariableName) * 2 - - token(matchType + xlength<<lengthShift + xoffset) + _ = a*b + c + _ = a - b - c + _ = a - (b - c) + _ = a - b*c + _ = a - (b * c) + _ = a * b / c + _ = a / *b + _ = x[a|^b] + _ = x[a / *b] + _ = a & ^b + _ = a + +b + _ = a - -b + _ = x[a*-b] + _ = x[a + +b] + _ = x ^ y ^ z + _ = b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF] + _ = len(longVariableName) * 2 + + _ = token(matchType + xlength<<lengthShift + xoffset) } @@ -625,7 +644,7 @@ func _() { func f() { // os.Open parameters should remain on two lines if writer, err = os.Open(outfile, s.O_WRONLY|os.O_CREATE| - os.O_TRUNC,0666); err != nil { + os.O_TRUNC, 0666); err != nil { log.Fatal(err) } } diff --git a/src/pkg/go/printer/testdata/parser.go b/src/pkg/go/printer/testdata/parser.go new file mode 100644 index 000000000..5c57e41d1 --- /dev/null +++ b/src/pkg/go/printer/testdata/parser.go @@ -0,0 +1,2252 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package parser implements a parser for Go source files. Input may be +// provided in a variety of forms (see the various Parse* functions); the +// output is an abstract syntax tree (AST) representing the Go source. The +// parser is invoked through one of the Parse* functions. +// +package parser + +import ( + "fmt" + "go/ast" + "go/scanner" + "go/token" +) + + +// The mode parameter to the Parse* functions is a set of flags (or 0). +// They control the amount of source code parsed and other optional +// parser functionality. +// +const ( + PackageClauseOnly uint = 1 << iota // parsing stops after package clause + ImportsOnly // parsing stops after import declarations + ParseComments // parse comments and add them to AST + Trace // print a trace of parsed productions + DeclarationErrors // report declaration errors +) + + +// The parser structure holds the parser's internal state. +type parser struct { + file *token.File + scanner.ErrorVector + scanner scanner.Scanner + + // Tracing/debugging + mode uint // parsing mode + trace bool // == (mode & Trace != 0) + indent uint // indentation used for tracing output + + // Comments + comments []*ast.CommentGroup + leadComment *ast.CommentGroup // last lead comment + lineComment *ast.CommentGroup // last line comment + + // Next token + pos token.Pos // token position + tok token.Token // one token look-ahead + lit string // token literal + + // Non-syntactic parser control + exprLev int // < 0: in control clause, >= 0: in expression + + // Ordinary identifer scopes + pkgScope *ast.Scope // pkgScope.Outer == nil + topScope *ast.Scope // top-most scope; may be pkgScope + unresolved []*ast.Ident // unresolved identifiers + imports []*ast.ImportSpec // list of imports + + // Label scope + // (maintained by open/close LabelScope) + labelScope *ast.Scope // label scope for current function + targetStack [][]*ast.Ident // stack of unresolved labels +} + + +// scannerMode returns the scanner mode bits given the parser's mode bits. +func scannerMode(mode uint) uint { + var m uint = scanner.InsertSemis + if mode&ParseComments != 0 { + m |= scanner.ScanComments + } + return m +} + + +func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode uint) { + p.file = fset.AddFile(filename, fset.Base(), len(src)) + p.scanner.Init(p.file, src, p, scannerMode(mode)) + + p.mode = mode + p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently) + + p.next() + + // set up the pkgScope here (as opposed to in parseFile) because + // there are other parser entry points (ParseExpr, etc.) + p.openScope() + p.pkgScope = p.topScope + + // for the same reason, set up a label scope + p.openLabelScope() +} + + +// ---------------------------------------------------------------------------- +// Scoping support + +func (p *parser) openScope() { + p.topScope = ast.NewScope(p.topScope) +} + + +func (p *parser) closeScope() { + p.topScope = p.topScope.Outer +} + + +func (p *parser) openLabelScope() { + p.labelScope = ast.NewScope(p.labelScope) + p.targetStack = append(p.targetStack, nil) +} + + +func (p *parser) closeLabelScope() { + // resolve labels + n := len(p.targetStack) - 1 + scope := p.labelScope + for _, ident := range p.targetStack[n] { + ident.Obj = scope.Lookup(ident.Name) + if ident.Obj == nil && p.mode&DeclarationErrors != 0 { + p.error(ident.Pos(), fmt.Sprintf("label %s undefined", ident.Name)) + } + } + // pop label scope + p.targetStack = p.targetStack[0:n] + p.labelScope = p.labelScope.Outer +} + + +func (p *parser) declare(decl interface{}, scope *ast.Scope, kind ast.ObjKind, idents ...*ast.Ident) { + for _, ident := range idents { + assert(ident.Obj == nil, "identifier already declared or resolved") + if ident.Name != "_" { + obj := ast.NewObj(kind, ident.Name) + // remember the corresponding declaration for redeclaration + // errors and global variable resolution/typechecking phase + obj.Decl = decl + if alt := scope.Insert(obj); alt != nil && p.mode&DeclarationErrors != 0 { + prevDecl := "" + if pos := alt.Pos(); pos.IsValid() { + prevDecl = fmt.Sprintf("\n\tprevious declaration at %s", p.file.Position(pos)) + } + p.error(ident.Pos(), fmt.Sprintf("%s redeclared in this block%s", ident.Name, prevDecl)) + } + ident.Obj = obj + } + } +} + + +func (p *parser) shortVarDecl(idents []*ast.Ident) { + // Go spec: A short variable declaration may redeclare variables + // provided they were originally declared in the same block with + // the same type, and at least one of the non-blank variables is new. + n := 0 // number of new variables + for _, ident := range idents { + assert(ident.Obj == nil, "identifier already declared or resolved") + if ident.Name != "_" { + obj := ast.NewObj(ast.Var, ident.Name) + // short var declarations cannot have redeclaration errors + // and are not global => no need to remember the respective + // declaration + alt := p.topScope.Insert(obj) + if alt == nil { + n++ // new declaration + alt = obj + } + ident.Obj = alt + } + } + if n == 0 && p.mode&DeclarationErrors != 0 { + p.error(idents[0].Pos(), "no new variables on left side of :=") + } +} + + +// The unresolved object is a sentinel to mark identifiers that have been added +// to the list of unresolved identifiers. The sentinel is only used for verifying +// internal consistency. +var unresolved = new(ast.Object) + + +func (p *parser) resolve(x ast.Expr) { + // nothing to do if x is not an identifier or the blank identifier + ident, _ := x.(*ast.Ident) + if ident == nil { + return + } + assert(ident.Obj == nil, "identifier already declared or resolved") + if ident.Name == "_" { + return + } + // try to resolve the identifier + for s := p.topScope; s != nil; s = s.Outer { + if obj := s.Lookup(ident.Name); obj != nil { + ident.Obj = obj + return + } + } + // all local scopes are known, so any unresolved identifier + // must be found either in the file scope, package scope + // (perhaps in another file), or universe scope --- collect + // them so that they can be resolved later + ident.Obj = unresolved + p.unresolved = append(p.unresolved, ident) +} + + +// ---------------------------------------------------------------------------- +// Parsing support + +func (p *parser) printTrace(a ...interface{}) { + const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + + ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + const n = uint(len(dots)) + pos := p.file.Position(p.pos) + fmt.Printf("%5d:%3d: ", pos.Line, pos.Column) + i := 2 * p.indent + for ; i > n; i -= n { + fmt.Print(dots) + } + fmt.Print(dots[0:i]) + fmt.Println(a...) +} + + +func trace(p *parser, msg string) *parser { + p.printTrace(msg, "(") + p.indent++ + return p +} + + +// Usage pattern: defer un(trace(p, "...")); +func un(p *parser) { + p.indent-- + p.printTrace(")") +} + + +// Advance to the next token. +func (p *parser) next0() { + // Because of one-token look-ahead, print the previous token + // when tracing as it provides a more readable output. The + // very first token (!p.pos.IsValid()) is not initialized + // (it is token.ILLEGAL), so don't print it . + if p.trace && p.pos.IsValid() { + s := p.tok.String() + switch { + case p.tok.IsLiteral(): + p.printTrace(s, p.lit) + case p.tok.IsOperator(), p.tok.IsKeyword(): + p.printTrace("\"" + s + "\"") + default: + p.printTrace(s) + } + } + + p.pos, p.tok, p.lit = p.scanner.Scan() +} + +// Consume a comment and return it and the line on which it ends. +func (p *parser) consumeComment() (comment *ast.Comment, endline int) { + // /*-style comments may end on a different line than where they start. + // Scan the comment for '\n' chars and adjust endline accordingly. + endline = p.file.Line(p.pos) + if p.lit[1] == '*' { + // don't use range here - no need to decode Unicode code points + for i := 0; i < len(p.lit); i++ { + if p.lit[i] == '\n' { + endline++ + } + } + } + + comment = &ast.Comment{p.pos, p.lit} + p.next0() + + return +} + + +// Consume a group of adjacent comments, add it to the parser's +// comments list, and return it together with the line at which +// the last comment in the group ends. An empty line or non-comment +// token terminates a comment group. +// +func (p *parser) consumeCommentGroup() (comments *ast.CommentGroup, endline int) { + var list []*ast.Comment + endline = p.file.Line(p.pos) + for p.tok == token.COMMENT && endline+1 >= p.file.Line(p.pos) { + var comment *ast.Comment + comment, endline = p.consumeComment() + list = append(list, comment) + } + + // add comment group to the comments list + comments = &ast.CommentGroup{list} + p.comments = append(p.comments, comments) + + return +} + + +// Advance to the next non-comment token. In the process, collect +// any comment groups encountered, and remember the last lead and +// and line comments. +// +// A lead comment is a comment group that starts and ends in a +// line without any other tokens and that is followed by a non-comment +// token on the line immediately after the comment group. +// +// A line comment is a comment group that follows a non-comment +// token on the same line, and that has no tokens after it on the line +// where it ends. +// +// Lead and line comments may be considered documentation that is +// stored in the AST. +// +func (p *parser) next() { + p.leadComment = nil + p.lineComment = nil + line := p.file.Line(p.pos) // current line + p.next0() + + if p.tok == token.COMMENT { + var comment *ast.CommentGroup + var endline int + + if p.file.Line(p.pos) == line { + // The comment is on same line as the previous token; it + // cannot be a lead comment but may be a line comment. + comment, endline = p.consumeCommentGroup() + if p.file.Line(p.pos) != endline { + // The next token is on a different line, thus + // the last comment group is a line comment. + p.lineComment = comment + } + } + + // consume successor comments, if any + endline = -1 + for p.tok == token.COMMENT { + comment, endline = p.consumeCommentGroup() + } + + if endline+1 == p.file.Line(p.pos) { + // The next token is following on the line immediately after the + // comment group, thus the last comment group is a lead comment. + p.leadComment = comment + } + } +} + + +func (p *parser) error(pos token.Pos, msg string) { + p.Error(p.file.Position(pos), msg) +} + + +func (p *parser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if pos == p.pos { + // the error happened at the current position; + // make the error message more specific + if p.tok == token.SEMICOLON && p.lit[0] == '\n' { + msg += ", found newline" + } else { + msg += ", found '" + p.tok.String() + "'" + if p.tok.IsLiteral() { + msg += " " + p.lit + } + } + } + p.error(pos, msg) +} + + +func (p *parser) expect(tok token.Token) token.Pos { + pos := p.pos + if p.tok != tok { + p.errorExpected(pos, "'"+tok.String()+"'") + } + p.next() // make progress + return pos +} + + +func (p *parser) expectSemi() { + if p.tok != token.RPAREN && p.tok != token.RBRACE { + p.expect(token.SEMICOLON) + } +} + + +func assert(cond bool, msg string) { + if !cond { + panic("go/parser internal error: " + msg) + } +} + + +// ---------------------------------------------------------------------------- +// Identifiers + +func (p *parser) parseIdent() *ast.Ident { + pos := p.pos + name := "_" + if p.tok == token.IDENT { + name = p.lit + p.next() + } else { + p.expect(token.IDENT) // use expect() error handling + } + return &ast.Ident{pos, name, nil} +} + + +func (p *parser) parseIdentList() (list []*ast.Ident) { + if p.trace { + defer un(trace(p, "IdentList")) + } + + list = append(list, p.parseIdent()) + for p.tok == token.COMMA { + p.next() + list = append(list, p.parseIdent()) + } + + return +} + + +// ---------------------------------------------------------------------------- +// Common productions + +// If lhs is set, result list elements which are identifiers are not resolved. +func (p *parser) parseExprList(lhs bool) (list []ast.Expr) { + if p.trace { + defer un(trace(p, "ExpressionList")) + } + + list = append(list, p.parseExpr(lhs)) + for p.tok == token.COMMA { + p.next() + list = append(list, p.parseExpr(lhs)) + } + + return +} + + +func (p *parser) parseLhsList() []ast.Expr { + list := p.parseExprList(true) + switch p.tok { + case token.DEFINE: + // lhs of a short variable declaration + p.shortVarDecl(p.makeIdentList(list)) + case token.COLON: + // lhs of a label declaration or a communication clause of a select + // statement (parseLhsList is not called when parsing the case clause + // of a switch statement): + // - labels are declared by the caller of parseLhsList + // - for communication clauses, if there is a stand-alone identifier + // followed by a colon, we have a syntax error; there is no need + // to resolve the identifier in that case + default: + // identifiers must be declared elsewhere + for _, x := range list { + p.resolve(x) + } + } + return list +} + + +func (p *parser) parseRhsList() []ast.Expr { + return p.parseExprList(false) +} + + +// ---------------------------------------------------------------------------- +// Types + +func (p *parser) parseType() ast.Expr { + if p.trace { + defer un(trace(p, "Type")) + } + + typ := p.tryType() + + if typ == nil { + pos := p.pos + p.errorExpected(pos, "type") + p.next() // make progress + return &ast.BadExpr{pos, p.pos} + } + + return typ +} + + +// If the result is an identifier, it is not resolved. +func (p *parser) parseTypeName() ast.Expr { + if p.trace { + defer un(trace(p, "TypeName")) + } + + ident := p.parseIdent() + // don't resolve ident yet - it may be a parameter or field name + + if p.tok == token.PERIOD { + // ident is a package name + p.next() + p.resolve(ident) + sel := p.parseIdent() + return &ast.SelectorExpr{ident, sel} + } + + return ident +} + + +func (p *parser) parseArrayType(ellipsisOk bool) ast.Expr { + if p.trace { + defer un(trace(p, "ArrayType")) + } + + lbrack := p.expect(token.LBRACK) + var len ast.Expr + if ellipsisOk && p.tok == token.ELLIPSIS { + len = &ast.Ellipsis{p.pos, nil} + p.next() + } else if p.tok != token.RBRACK { + len = p.parseRhs() + } + p.expect(token.RBRACK) + elt := p.parseType() + + return &ast.ArrayType{lbrack, len, elt} +} + + +func (p *parser) makeIdentList(list []ast.Expr) []*ast.Ident { + idents := make([]*ast.Ident, len(list)) + for i, x := range list { + ident, isIdent := x.(*ast.Ident) + if !isIdent { + pos := x.(ast.Expr).Pos() + p.errorExpected(pos, "identifier") + ident = &ast.Ident{pos, "_", nil} + } + idents[i] = ident + } + return idents +} + + +func (p *parser) parseFieldDecl(scope *ast.Scope) *ast.Field { + if p.trace { + defer un(trace(p, "FieldDecl")) + } + + doc := p.leadComment + + // fields + list, typ := p.parseVarList(false) + + // optional tag + var tag *ast.BasicLit + if p.tok == token.STRING { + tag = &ast.BasicLit{p.pos, p.tok, p.lit} + p.next() + } + + // analyze case + var idents []*ast.Ident + if typ != nil { + // IdentifierList Type + idents = p.makeIdentList(list) + } else { + // ["*"] TypeName (AnonymousField) + typ = list[0] // we always have at least one element + p.resolve(typ) + if n := len(list); n > 1 || !isTypeName(deref(typ)) { + pos := typ.Pos() + p.errorExpected(pos, "anonymous field") + typ = &ast.BadExpr{pos, list[n-1].End()} + } + } + + p.expectSemi() // call before accessing p.linecomment + + field := &ast.Field{doc, idents, typ, tag, p.lineComment} + p.declare(field, scope, ast.Var, idents...) + + return field +} + + +func (p *parser) parseStructType() *ast.StructType { + if p.trace { + defer un(trace(p, "StructType")) + } + + pos := p.expect(token.STRUCT) + lbrace := p.expect(token.LBRACE) + scope := ast.NewScope(nil) // struct scope + var list []*ast.Field + for p.tok == token.IDENT || p.tok == token.MUL || p.tok == token.LPAREN { + // a field declaration cannot start with a '(' but we accept + // it here for more robust parsing and better error messages + // (parseFieldDecl will check and complain if necessary) + list = append(list, p.parseFieldDecl(scope)) + } + rbrace := p.expect(token.RBRACE) + + // TODO(gri): store struct scope in AST + return &ast.StructType{pos, &ast.FieldList{lbrace, list, rbrace}, false} +} + + +func (p *parser) parsePointerType() *ast.StarExpr { + if p.trace { + defer un(trace(p, "PointerType")) + } + + star := p.expect(token.MUL) + base := p.parseType() + + return &ast.StarExpr{star, base} +} + + +func (p *parser) tryVarType(isParam bool) ast.Expr { + if isParam && p.tok == token.ELLIPSIS { + pos := p.pos + p.next() + typ := p.tryIdentOrType(isParam) // don't use parseType so we can provide better error message + if typ == nil { + p.error(pos, "'...' parameter is missing type") + typ = &ast.BadExpr{pos, p.pos} + } + if p.tok != token.RPAREN { + p.error(pos, "can use '...' with last parameter type only") + } + return &ast.Ellipsis{pos, typ} + } + return p.tryIdentOrType(false) +} + + +func (p *parser) parseVarType(isParam bool) ast.Expr { + typ := p.tryVarType(isParam) + if typ == nil { + pos := p.pos + p.errorExpected(pos, "type") + p.next() // make progress + typ = &ast.BadExpr{pos, p.pos} + } + return typ +} + + +func (p *parser) parseVarList(isParam bool) (list []ast.Expr, typ ast.Expr) { + if p.trace { + defer un(trace(p, "VarList")) + } + + // a list of identifiers looks like a list of type names + for { + // parseVarType accepts any type (including parenthesized ones) + // even though the syntax does not permit them here: we + // accept them all for more robust parsing and complain + // afterwards + list = append(list, p.parseVarType(isParam)) + if p.tok != token.COMMA { + break + } + p.next() + } + + // if we had a list of identifiers, it must be followed by a type + typ = p.tryVarType(isParam) + if typ != nil { + p.resolve(typ) + } + + return +} + + +func (p *parser) parseParameterList(scope *ast.Scope, ellipsisOk bool) (params []*ast.Field) { + if p.trace { + defer un(trace(p, "ParameterList")) + } + + list, typ := p.parseVarList(ellipsisOk) + if typ != nil { + // IdentifierList Type + idents := p.makeIdentList(list) + field := &ast.Field{nil, idents, typ, nil, nil} + params = append(params, field) + // Go spec: The scope of an identifier denoting a function + // parameter or result variable is the function body. + p.declare(field, scope, ast.Var, idents...) + if p.tok == token.COMMA { + p.next() + } + + for p.tok != token.RPAREN && p.tok != token.EOF { + idents := p.parseIdentList() + typ := p.parseVarType(ellipsisOk) + field := &ast.Field{nil, idents, typ, nil, nil} + params = append(params, field) + // Go spec: The scope of an identifier denoting a function + // parameter or result variable is the function body. + p.declare(field, scope, ast.Var, idents...) + if p.tok != token.COMMA { + break + } + p.next() + } + + } else { + // Type { "," Type } (anonymous parameters) + params = make([]*ast.Field, len(list)) + for i, x := range list { + p.resolve(x) + params[i] = &ast.Field{Type: x} + } + } + + return +} + + +func (p *parser) parseParameters(scope *ast.Scope, ellipsisOk bool) *ast.FieldList { + if p.trace { + defer un(trace(p, "Parameters")) + } + + var params []*ast.Field + lparen := p.expect(token.LPAREN) + if p.tok != token.RPAREN { + params = p.parseParameterList(scope, ellipsisOk) + } + rparen := p.expect(token.RPAREN) + + return &ast.FieldList{lparen, params, rparen} +} + + +func (p *parser) parseResult(scope *ast.Scope) *ast.FieldList { + if p.trace { + defer un(trace(p, "Result")) + } + + if p.tok == token.LPAREN { + return p.parseParameters(scope, false) + } + + typ := p.tryType() + if typ != nil { + list := make([]*ast.Field, 1) + list[0] = &ast.Field{Type: typ} + return &ast.FieldList{List: list} + } + + return nil +} + + +func (p *parser) parseSignature(scope *ast.Scope) (params, results *ast.FieldList) { + if p.trace { + defer un(trace(p, "Signature")) + } + + params = p.parseParameters(scope, true) + results = p.parseResult(scope) + + return +} + + +func (p *parser) parseFuncType() (*ast.FuncType, *ast.Scope) { + if p.trace { + defer un(trace(p, "FuncType")) + } + + pos := p.expect(token.FUNC) + scope := ast.NewScope(p.topScope) // function scope + params, results := p.parseSignature(scope) + + return &ast.FuncType{pos, params, results}, scope +} + + +func (p *parser) parseMethodSpec(scope *ast.Scope) *ast.Field { + if p.trace { + defer un(trace(p, "MethodSpec")) + } + + doc := p.leadComment + var idents []*ast.Ident + var typ ast.Expr + x := p.parseTypeName() + if ident, isIdent := x.(*ast.Ident); isIdent && p.tok == token.LPAREN { + // method + idents = []*ast.Ident{ident} + scope := ast.NewScope(nil) // method scope + params, results := p.parseSignature(scope) + typ = &ast.FuncType{token.NoPos, params, results} + } else { + // embedded interface + typ = x + } + p.expectSemi() // call before accessing p.linecomment + + spec := &ast.Field{doc, idents, typ, nil, p.lineComment} + p.declare(spec, scope, ast.Fun, idents...) + + return spec +} + + +func (p *parser) parseInterfaceType() *ast.InterfaceType { + if p.trace { + defer un(trace(p, "InterfaceType")) + } + + pos := p.expect(token.INTERFACE) + lbrace := p.expect(token.LBRACE) + scope := ast.NewScope(nil) // interface scope + var list []*ast.Field + for p.tok == token.IDENT { + list = append(list, p.parseMethodSpec(scope)) + } + rbrace := p.expect(token.RBRACE) + + // TODO(gri): store interface scope in AST + return &ast.InterfaceType{pos, &ast.FieldList{lbrace, list, rbrace}, false} +} + + +func (p *parser) parseMapType() *ast.MapType { + if p.trace { + defer un(trace(p, "MapType")) + } + + pos := p.expect(token.MAP) + p.expect(token.LBRACK) + key := p.parseType() + p.expect(token.RBRACK) + value := p.parseType() + + return &ast.MapType{pos, key, value} +} + + +func (p *parser) parseChanType() *ast.ChanType { + if p.trace { + defer un(trace(p, "ChanType")) + } + + pos := p.pos + dir := ast.SEND | ast.RECV + if p.tok == token.CHAN { + p.next() + if p.tok == token.ARROW { + p.next() + dir = ast.SEND + } + } else { + p.expect(token.ARROW) + p.expect(token.CHAN) + dir = ast.RECV + } + value := p.parseType() + + return &ast.ChanType{pos, dir, value} +} + + +// If the result is an identifier, it is not resolved. +func (p *parser) tryIdentOrType(ellipsisOk bool) ast.Expr { + switch p.tok { + case token.IDENT: + return p.parseTypeName() + case token.LBRACK: + return p.parseArrayType(ellipsisOk) + case token.STRUCT: + return p.parseStructType() + case token.MUL: + return p.parsePointerType() + case token.FUNC: + typ, _ := p.parseFuncType() + return typ + case token.INTERFACE: + return p.parseInterfaceType() + case token.MAP: + return p.parseMapType() + case token.CHAN, token.ARROW: + return p.parseChanType() + case token.LPAREN: + lparen := p.pos + p.next() + typ := p.parseType() + rparen := p.expect(token.RPAREN) + return &ast.ParenExpr{lparen, typ, rparen} + } + + // no type found + return nil +} + + +func (p *parser) tryType() ast.Expr { + typ := p.tryIdentOrType(false) + if typ != nil { + p.resolve(typ) + } + return typ +} + + +// ---------------------------------------------------------------------------- +// Blocks + +func (p *parser) parseStmtList() (list []ast.Stmt) { + if p.trace { + defer un(trace(p, "StatementList")) + } + + for p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF { + list = append(list, p.parseStmt()) + } + + return +} + + +func (p *parser) parseBody(scope *ast.Scope) *ast.BlockStmt { + if p.trace { + defer un(trace(p, "Body")) + } + + lbrace := p.expect(token.LBRACE) + p.topScope = scope // open function scope + p.openLabelScope() + list := p.parseStmtList() + p.closeLabelScope() + p.closeScope() + rbrace := p.expect(token.RBRACE) + + return &ast.BlockStmt{lbrace, list, rbrace} +} + + +func (p *parser) parseBlockStmt() *ast.BlockStmt { + if p.trace { + defer un(trace(p, "BlockStmt")) + } + + lbrace := p.expect(token.LBRACE) + p.openScope() + list := p.parseStmtList() + p.closeScope() + rbrace := p.expect(token.RBRACE) + + return &ast.BlockStmt{lbrace, list, rbrace} +} + + +// ---------------------------------------------------------------------------- +// Expressions + +func (p *parser) parseFuncTypeOrLit() ast.Expr { + if p.trace { + defer un(trace(p, "FuncTypeOrLit")) + } + + typ, scope := p.parseFuncType() + if p.tok != token.LBRACE { + // function type only + return typ + } + + p.exprLev++ + body := p.parseBody(scope) + p.exprLev-- + + return &ast.FuncLit{typ, body} +} + + +// parseOperand may return an expression or a raw type (incl. array +// types of the form [...]T. Callers must verify the result. +// If lhs is set and the result is an identifier, it is not resolved. +// +func (p *parser) parseOperand(lhs bool) ast.Expr { + if p.trace { + defer un(trace(p, "Operand")) + } + + switch p.tok { + case token.IDENT: + x := p.parseIdent() + if !lhs { + p.resolve(x) + } + return x + + case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING: + x := &ast.BasicLit{p.pos, p.tok, p.lit} + p.next() + return x + + case token.LPAREN: + lparen := p.pos + p.next() + p.exprLev++ + x := p.parseRhs() + p.exprLev-- + rparen := p.expect(token.RPAREN) + return &ast.ParenExpr{lparen, x, rparen} + + case token.FUNC: + return p.parseFuncTypeOrLit() + + default: + if typ := p.tryIdentOrType(true); typ != nil { + // could be type for composite literal or conversion + _, isIdent := typ.(*ast.Ident) + assert(!isIdent, "type cannot be identifier") + return typ + } + } + + pos := p.pos + p.errorExpected(pos, "operand") + p.next() // make progress + return &ast.BadExpr{pos, p.pos} +} + + +func (p *parser) parseSelector(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "Selector")) + } + + sel := p.parseIdent() + + return &ast.SelectorExpr{x, sel} +} + + +func (p *parser) parseTypeAssertion(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "TypeAssertion")) + } + + p.expect(token.LPAREN) + var typ ast.Expr + if p.tok == token.TYPE { + // type switch: typ == nil + p.next() + } else { + typ = p.parseType() + } + p.expect(token.RPAREN) + + return &ast.TypeAssertExpr{x, typ} +} + + +func (p *parser) parseIndexOrSlice(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "IndexOrSlice")) + } + + lbrack := p.expect(token.LBRACK) + p.exprLev++ + var low, high ast.Expr + isSlice := false + if p.tok != token.COLON { + low = p.parseRhs() + } + if p.tok == token.COLON { + isSlice = true + p.next() + if p.tok != token.RBRACK { + high = p.parseRhs() + } + } + p.exprLev-- + rbrack := p.expect(token.RBRACK) + + if isSlice { + return &ast.SliceExpr{x, lbrack, low, high, rbrack} + } + return &ast.IndexExpr{x, lbrack, low, rbrack} +} + + +func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr { + if p.trace { + defer un(trace(p, "CallOrConversion")) + } + + lparen := p.expect(token.LPAREN) + p.exprLev++ + var list []ast.Expr + var ellipsis token.Pos + for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() { + list = append(list, p.parseRhs()) + if p.tok == token.ELLIPSIS { + ellipsis = p.pos + p.next() + } + if p.tok != token.COMMA { + break + } + p.next() + } + p.exprLev-- + rparen := p.expect(token.RPAREN) + + return &ast.CallExpr{fun, lparen, list, ellipsis, rparen} +} + + +func (p *parser) parseElement(keyOk bool) ast.Expr { + if p.trace { + defer un(trace(p, "Element")) + } + + if p.tok == token.LBRACE { + return p.parseLiteralValue(nil) + } + + x := p.parseExpr(keyOk) // don't resolve if map key + if keyOk { + if p.tok == token.COLON { + colon := p.pos + p.next() + return &ast.KeyValueExpr{x, colon, p.parseElement(false)} + } + p.resolve(x) // not a map key + } + + return x +} + + +func (p *parser) parseElementList() (list []ast.Expr) { + if p.trace { + defer un(trace(p, "ElementList")) + } + + for p.tok != token.RBRACE && p.tok != token.EOF { + list = append(list, p.parseElement(true)) + if p.tok != token.COMMA { + break + } + p.next() + } + + return +} + + +func (p *parser) parseLiteralValue(typ ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "LiteralValue")) + } + + lbrace := p.expect(token.LBRACE) + var elts []ast.Expr + p.exprLev++ + if p.tok != token.RBRACE { + elts = p.parseElementList() + } + p.exprLev-- + rbrace := p.expect(token.RBRACE) + return &ast.CompositeLit{typ, lbrace, elts, rbrace} +} + + +// checkExpr checks that x is an expression (and not a type). +func (p *parser) checkExpr(x ast.Expr) ast.Expr { + switch t := unparen(x).(type) { + case *ast.BadExpr: + case *ast.Ident: + case *ast.BasicLit: + case *ast.FuncLit: + case *ast.CompositeLit: + case *ast.ParenExpr: + panic("unreachable") + case *ast.SelectorExpr: + case *ast.IndexExpr: + case *ast.SliceExpr: + case *ast.TypeAssertExpr: + if t.Type == nil { + // the form X.(type) is only allowed in type switch expressions + p.errorExpected(x.Pos(), "expression") + x = &ast.BadExpr{x.Pos(), x.End()} + } + case *ast.CallExpr: + case *ast.StarExpr: + case *ast.UnaryExpr: + if t.Op == token.RANGE { + // the range operator is only allowed at the top of a for statement + p.errorExpected(x.Pos(), "expression") + x = &ast.BadExpr{x.Pos(), x.End()} + } + case *ast.BinaryExpr: + default: + // all other nodes are not proper expressions + p.errorExpected(x.Pos(), "expression") + x = &ast.BadExpr{x.Pos(), x.End()} + } + return x +} + + +// isTypeName returns true iff x is a (qualified) TypeName. +func isTypeName(x ast.Expr) bool { + switch t := x.(type) { + case *ast.BadExpr: + case *ast.Ident: + case *ast.SelectorExpr: + _, isIdent := t.X.(*ast.Ident) + return isIdent + default: + return false // all other nodes are not type names + } + return true +} + + +// isLiteralType returns true iff x is a legal composite literal type. +func isLiteralType(x ast.Expr) bool { + switch t := x.(type) { + case *ast.BadExpr: + case *ast.Ident: + case *ast.SelectorExpr: + _, isIdent := t.X.(*ast.Ident) + return isIdent + case *ast.ArrayType: + case *ast.StructType: + case *ast.MapType: + default: + return false // all other nodes are not legal composite literal types + } + return true +} + + +// If x is of the form *T, deref returns T, otherwise it returns x. +func deref(x ast.Expr) ast.Expr { + if p, isPtr := x.(*ast.StarExpr); isPtr { + x = p.X + } + return x +} + + +// If x is of the form (T), unparen returns unparen(T), otherwise it returns x. +func unparen(x ast.Expr) ast.Expr { + if p, isParen := x.(*ast.ParenExpr); isParen { + x = unparen(p.X) + } + return x +} + + +// checkExprOrType checks that x is an expression or a type +// (and not a raw type such as [...]T). +// +func (p *parser) checkExprOrType(x ast.Expr) ast.Expr { + switch t := unparen(x).(type) { + case *ast.ParenExpr: + panic("unreachable") + case *ast.UnaryExpr: + if t.Op == token.RANGE { + // the range operator is only allowed at the top of a for statement + p.errorExpected(x.Pos(), "expression") + x = &ast.BadExpr{x.Pos(), x.End()} + } + case *ast.ArrayType: + if len, isEllipsis := t.Len.(*ast.Ellipsis); isEllipsis { + p.error(len.Pos(), "expected array length, found '...'") + x = &ast.BadExpr{x.Pos(), x.End()} + } + } + + // all other nodes are expressions or types + return x +} + + +// If lhs is set and the result is an identifier, it is not resolved. +func (p *parser) parsePrimaryExpr(lhs bool) ast.Expr { + if p.trace { + defer un(trace(p, "PrimaryExpr")) + } + + x := p.parseOperand(lhs) +L: + for { + switch p.tok { + case token.PERIOD: + p.next() + if lhs { + p.resolve(x) + } + switch p.tok { + case token.IDENT: + x = p.parseSelector(p.checkExpr(x)) + case token.LPAREN: + x = p.parseTypeAssertion(p.checkExpr(x)) + default: + pos := p.pos + p.next() // make progress + p.errorExpected(pos, "selector or type assertion") + x = &ast.BadExpr{pos, p.pos} + } + case token.LBRACK: + if lhs { + p.resolve(x) + } + x = p.parseIndexOrSlice(p.checkExpr(x)) + case token.LPAREN: + if lhs { + p.resolve(x) + } + x = p.parseCallOrConversion(p.checkExprOrType(x)) + case token.LBRACE: + if isLiteralType(x) && (p.exprLev >= 0 || !isTypeName(x)) { + if lhs { + p.resolve(x) + } + x = p.parseLiteralValue(x) + } else { + break L + } + default: + break L + } + lhs = false // no need to try to resolve again + } + + return x +} + + +// If lhs is set and the result is an identifier, it is not resolved. +func (p *parser) parseUnaryExpr(lhs bool) ast.Expr { + if p.trace { + defer un(trace(p, "UnaryExpr")) + } + + switch p.tok { + case token.ADD, token.SUB, token.NOT, token.XOR, token.AND, token.RANGE: + pos, op := p.pos, p.tok + p.next() + x := p.parseUnaryExpr(false) + return &ast.UnaryExpr{pos, op, p.checkExpr(x)} + + case token.ARROW: + // channel type or receive expression + pos := p.pos + p.next() + if p.tok == token.CHAN { + p.next() + value := p.parseType() + return &ast.ChanType{pos, ast.RECV, value} + } + + x := p.parseUnaryExpr(false) + return &ast.UnaryExpr{pos, token.ARROW, p.checkExpr(x)} + + case token.MUL: + // pointer type or unary "*" expression + pos := p.pos + p.next() + x := p.parseUnaryExpr(false) + return &ast.StarExpr{pos, p.checkExprOrType(x)} + } + + return p.parsePrimaryExpr(lhs) +} + + +// If lhs is set and the result is an identifier, it is not resolved. +func (p *parser) parseBinaryExpr(lhs bool, prec1 int) ast.Expr { + if p.trace { + defer un(trace(p, "BinaryExpr")) + } + + x := p.parseUnaryExpr(lhs) + for prec := p.tok.Precedence(); prec >= prec1; prec-- { + for p.tok.Precedence() == prec { + pos, op := p.pos, p.tok + p.next() + if lhs { + p.resolve(x) + lhs = false + } + y := p.parseBinaryExpr(false, prec+1) + x = &ast.BinaryExpr{p.checkExpr(x), pos, op, p.checkExpr(y)} + } + } + + return x +} + + +// If lhs is set and the result is an identifier, it is not resolved. +// TODO(gri): parseExpr may return a type or even a raw type ([..]int) - +// should reject when a type/raw type is obviously not allowed +func (p *parser) parseExpr(lhs bool) ast.Expr { + if p.trace { + defer un(trace(p, "Expression")) + } + + return p.parseBinaryExpr(lhs, token.LowestPrec+1) +} + + +func (p *parser) parseRhs() ast.Expr { + return p.parseExpr(false) +} + + +// ---------------------------------------------------------------------------- +// Statements + +func (p *parser) parseSimpleStmt(labelOk bool) ast.Stmt { + if p.trace { + defer un(trace(p, "SimpleStmt")) + } + + x := p.parseLhsList() + + switch p.tok { + case + token.DEFINE, token.ASSIGN, token.ADD_ASSIGN, + token.SUB_ASSIGN, token.MUL_ASSIGN, token.QUO_ASSIGN, + token.REM_ASSIGN, token.AND_ASSIGN, token.OR_ASSIGN, + token.XOR_ASSIGN, token.SHL_ASSIGN, token.SHR_ASSIGN, token.AND_NOT_ASSIGN: + // assignment statement + pos, tok := p.pos, p.tok + p.next() + y := p.parseRhsList() + return &ast.AssignStmt{x, pos, tok, y} + } + + if len(x) > 1 { + p.errorExpected(x[0].Pos(), "1 expression") + // continue with first expression + } + + switch p.tok { + case token.COLON: + // labeled statement + colon := p.pos + p.next() + if label, isIdent := x[0].(*ast.Ident); labelOk && isIdent { + // Go spec: The scope of a label is the body of the function + // in which it is declared and excludes the body of any nested + // function. + stmt := &ast.LabeledStmt{label, colon, p.parseStmt()} + p.declare(stmt, p.labelScope, ast.Lbl, label) + return stmt + } + p.error(x[0].Pos(), "illegal label declaration") + return &ast.BadStmt{x[0].Pos(), colon + 1} + + case token.ARROW: + // send statement + arrow := p.pos + p.next() // consume "<-" + y := p.parseRhs() + return &ast.SendStmt{x[0], arrow, y} + + case token.INC, token.DEC: + // increment or decrement + s := &ast.IncDecStmt{x[0], p.pos, p.tok} + p.next() // consume "++" or "--" + return s + } + + // expression + return &ast.ExprStmt{x[0]} +} + + +func (p *parser) parseCallExpr() *ast.CallExpr { + x := p.parseRhs() + if call, isCall := x.(*ast.CallExpr); isCall { + return call + } + p.errorExpected(x.Pos(), "function/method call") + return nil +} + + +func (p *parser) parseGoStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "GoStmt")) + } + + pos := p.expect(token.GO) + call := p.parseCallExpr() + p.expectSemi() + if call == nil { + return &ast.BadStmt{pos, pos + 2} // len("go") + } + + return &ast.GoStmt{pos, call} +} + + +func (p *parser) parseDeferStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "DeferStmt")) + } + + pos := p.expect(token.DEFER) + call := p.parseCallExpr() + p.expectSemi() + if call == nil { + return &ast.BadStmt{pos, pos + 5} // len("defer") + } + + return &ast.DeferStmt{pos, call} +} + + +func (p *parser) parseReturnStmt() *ast.ReturnStmt { + if p.trace { + defer un(trace(p, "ReturnStmt")) + } + + pos := p.pos + p.expect(token.RETURN) + var x []ast.Expr + if p.tok != token.SEMICOLON && p.tok != token.RBRACE { + x = p.parseRhsList() + } + p.expectSemi() + + return &ast.ReturnStmt{pos, x} +} + + +func (p *parser) parseBranchStmt(tok token.Token) *ast.BranchStmt { + if p.trace { + defer un(trace(p, "BranchStmt")) + } + + pos := p.expect(tok) + var label *ast.Ident + if tok != token.FALLTHROUGH && p.tok == token.IDENT { + label = p.parseIdent() + // add to list of unresolved targets + n := len(p.targetStack) - 1 + p.targetStack[n] = append(p.targetStack[n], label) + } + p.expectSemi() + + return &ast.BranchStmt{pos, tok, label} +} + + +func (p *parser) makeExpr(s ast.Stmt) ast.Expr { + if s == nil { + return nil + } + if es, isExpr := s.(*ast.ExprStmt); isExpr { + return p.checkExpr(es.X) + } + p.error(s.Pos(), "expected condition, found simple statement") + return &ast.BadExpr{s.Pos(), s.End()} +} + + +func (p *parser) parseIfStmt() *ast.IfStmt { + if p.trace { + defer un(trace(p, "IfStmt")) + } + + pos := p.expect(token.IF) + p.openScope() + defer p.closeScope() + + var s ast.Stmt + var x ast.Expr + { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok == token.SEMICOLON { + p.next() + x = p.parseRhs() + } else { + s = p.parseSimpleStmt(false) + if p.tok == token.SEMICOLON { + p.next() + x = p.parseRhs() + } else { + x = p.makeExpr(s) + s = nil + } + } + p.exprLev = prevLev + } + + body := p.parseBlockStmt() + var else_ ast.Stmt + if p.tok == token.ELSE { + p.next() + else_ = p.parseStmt() + } else { + p.expectSemi() + } + + return &ast.IfStmt{pos, s, x, body, else_} +} + + +func (p *parser) parseTypeList() (list []ast.Expr) { + if p.trace { + defer un(trace(p, "TypeList")) + } + + list = append(list, p.parseType()) + for p.tok == token.COMMA { + p.next() + list = append(list, p.parseType()) + } + + return +} + + +func (p *parser) parseCaseClause(exprSwitch bool) *ast.CaseClause { + if p.trace { + defer un(trace(p, "CaseClause")) + } + + pos := p.pos + var list []ast.Expr + if p.tok == token.CASE { + p.next() + if exprSwitch { + list = p.parseRhsList() + } else { + list = p.parseTypeList() + } + } else { + p.expect(token.DEFAULT) + } + + colon := p.expect(token.COLON) + p.openScope() + body := p.parseStmtList() + p.closeScope() + + return &ast.CaseClause{pos, list, colon, body} +} + + +func isExprSwitch(s ast.Stmt) bool { + if s == nil { + return true + } + if e, ok := s.(*ast.ExprStmt); ok { + if a, ok := e.X.(*ast.TypeAssertExpr); ok { + return a.Type != nil // regular type assertion + } + return true + } + return false +} + + +func (p *parser) parseSwitchStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "SwitchStmt")) + } + + pos := p.expect(token.SWITCH) + p.openScope() + defer p.closeScope() + + var s1, s2 ast.Stmt + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + s2 = p.parseSimpleStmt(false) + } + if p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.LBRACE { + s2 = p.parseSimpleStmt(false) + } + } + p.exprLev = prevLev + } + + exprSwitch := isExprSwitch(s2) + lbrace := p.expect(token.LBRACE) + var list []ast.Stmt + for p.tok == token.CASE || p.tok == token.DEFAULT { + list = append(list, p.parseCaseClause(exprSwitch)) + } + rbrace := p.expect(token.RBRACE) + p.expectSemi() + body := &ast.BlockStmt{lbrace, list, rbrace} + + if exprSwitch { + return &ast.SwitchStmt{pos, s1, p.makeExpr(s2), body} + } + // type switch + // TODO(gri): do all the checks! + return &ast.TypeSwitchStmt{pos, s1, s2, body} +} + + +func (p *parser) parseCommClause() *ast.CommClause { + if p.trace { + defer un(trace(p, "CommClause")) + } + + p.openScope() + pos := p.pos + var comm ast.Stmt + if p.tok == token.CASE { + p.next() + lhs := p.parseLhsList() + if p.tok == token.ARROW { + // SendStmt + if len(lhs) > 1 { + p.errorExpected(lhs[0].Pos(), "1 expression") + // continue with first expression + } + arrow := p.pos + p.next() + rhs := p.parseRhs() + comm = &ast.SendStmt{lhs[0], arrow, rhs} + } else { + // RecvStmt + pos := p.pos + tok := p.tok + var rhs ast.Expr + if tok == token.ASSIGN || tok == token.DEFINE { + // RecvStmt with assignment + if len(lhs) > 2 { + p.errorExpected(lhs[0].Pos(), "1 or 2 expressions") + // continue with first two expressions + lhs = lhs[0:2] + } + p.next() + rhs = p.parseRhs() + } else { + // rhs must be single receive operation + if len(lhs) > 1 { + p.errorExpected(lhs[0].Pos(), "1 expression") + // continue with first expression + } + rhs = lhs[0] + lhs = nil // there is no lhs + } + if x, isUnary := rhs.(*ast.UnaryExpr); !isUnary || x.Op != token.ARROW { + p.errorExpected(rhs.Pos(), "send or receive operation") + rhs = &ast.BadExpr{rhs.Pos(), rhs.End()} + } + if lhs != nil { + comm = &ast.AssignStmt{lhs, pos, tok, []ast.Expr{rhs}} + } else { + comm = &ast.ExprStmt{rhs} + } + } + } else { + p.expect(token.DEFAULT) + } + + colon := p.expect(token.COLON) + body := p.parseStmtList() + p.closeScope() + + return &ast.CommClause{pos, comm, colon, body} +} + + +func (p *parser) parseSelectStmt() *ast.SelectStmt { + if p.trace { + defer un(trace(p, "SelectStmt")) + } + + pos := p.expect(token.SELECT) + lbrace := p.expect(token.LBRACE) + var list []ast.Stmt + for p.tok == token.CASE || p.tok == token.DEFAULT { + list = append(list, p.parseCommClause()) + } + rbrace := p.expect(token.RBRACE) + p.expectSemi() + body := &ast.BlockStmt{lbrace, list, rbrace} + + return &ast.SelectStmt{pos, body} +} + + +func (p *parser) parseForStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "ForStmt")) + } + + pos := p.expect(token.FOR) + p.openScope() + defer p.closeScope() + + var s1, s2, s3 ast.Stmt + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + s2 = p.parseSimpleStmt(false) + } + if p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.SEMICOLON { + s2 = p.parseSimpleStmt(false) + } + p.expectSemi() + if p.tok != token.LBRACE { + s3 = p.parseSimpleStmt(false) + } + } + p.exprLev = prevLev + } + + body := p.parseBlockStmt() + p.expectSemi() + + if as, isAssign := s2.(*ast.AssignStmt); isAssign { + // possibly a for statement with a range clause; check assignment operator + if as.Tok != token.ASSIGN && as.Tok != token.DEFINE { + p.errorExpected(as.TokPos, "'=' or ':='") + return &ast.BadStmt{pos, body.End()} + } + // check lhs + var key, value ast.Expr + switch len(as.Lhs) { + case 2: + key, value = as.Lhs[0], as.Lhs[1] + case 1: + key = as.Lhs[0] + default: + p.errorExpected(as.Lhs[0].Pos(), "1 or 2 expressions") + return &ast.BadStmt{pos, body.End()} + } + // check rhs + if len(as.Rhs) != 1 { + p.errorExpected(as.Rhs[0].Pos(), "1 expression") + return &ast.BadStmt{pos, body.End()} + } + if rhs, isUnary := as.Rhs[0].(*ast.UnaryExpr); isUnary && rhs.Op == token.RANGE { + // rhs is range expression + // (any short variable declaration was handled by parseSimpleStat above) + return &ast.RangeStmt{pos, key, value, as.TokPos, as.Tok, rhs.X, body} + } + p.errorExpected(s2.Pos(), "range clause") + return &ast.BadStmt{pos, body.End()} + } + + // regular for statement + return &ast.ForStmt{pos, s1, p.makeExpr(s2), s3, body} +} + + +func (p *parser) parseStmt() (s ast.Stmt) { + if p.trace { + defer un(trace(p, "Statement")) + } + + switch p.tok { + case token.CONST, token.TYPE, token.VAR: + s = &ast.DeclStmt{p.parseDecl()} + case + // tokens that may start a top-level expression + token.IDENT, token.INT, token.FLOAT, token.CHAR, token.STRING, token.FUNC, token.LPAREN, // operand + token.LBRACK, token.STRUCT, // composite type + token.MUL, token.AND, token.ARROW, token.ADD, token.SUB, token.XOR: // unary operators + s = p.parseSimpleStmt(true) + // because of the required look-ahead, labeled statements are + // parsed by parseSimpleStmt - don't expect a semicolon after + // them + if _, isLabeledStmt := s.(*ast.LabeledStmt); !isLabeledStmt { + p.expectSemi() + } + case token.GO: + s = p.parseGoStmt() + case token.DEFER: + s = p.parseDeferStmt() + case token.RETURN: + s = p.parseReturnStmt() + case token.BREAK, token.CONTINUE, token.GOTO, token.FALLTHROUGH: + s = p.parseBranchStmt(p.tok) + case token.LBRACE: + s = p.parseBlockStmt() + p.expectSemi() + case token.IF: + s = p.parseIfStmt() + case token.SWITCH: + s = p.parseSwitchStmt() + case token.SELECT: + s = p.parseSelectStmt() + case token.FOR: + s = p.parseForStmt() + case token.SEMICOLON: + s = &ast.EmptyStmt{p.pos} + p.next() + case token.RBRACE: + // a semicolon may be omitted before a closing "}" + s = &ast.EmptyStmt{p.pos} + default: + // no statement found + pos := p.pos + p.errorExpected(pos, "statement") + p.next() // make progress + s = &ast.BadStmt{pos, p.pos} + } + + return +} + + +// ---------------------------------------------------------------------------- +// Declarations + +type parseSpecFunction func(p *parser, doc *ast.CommentGroup, iota int) ast.Spec + + +func parseImportSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec { + if p.trace { + defer un(trace(p, "ImportSpec")) + } + + var ident *ast.Ident + switch p.tok { + case token.PERIOD: + ident = &ast.Ident{p.pos, ".", nil} + p.next() + case token.IDENT: + ident = p.parseIdent() + } + + var path *ast.BasicLit + if p.tok == token.STRING { + path = &ast.BasicLit{p.pos, p.tok, p.lit} + p.next() + } else { + p.expect(token.STRING) // use expect() error handling + } + p.expectSemi() // call before accessing p.linecomment + + // collect imports + spec := &ast.ImportSpec{doc, ident, path, p.lineComment} + p.imports = append(p.imports, spec) + + return spec +} + + +func parseConstSpec(p *parser, doc *ast.CommentGroup, iota int) ast.Spec { + if p.trace { + defer un(trace(p, "ConstSpec")) + } + + idents := p.parseIdentList() + typ := p.tryType() + var values []ast.Expr + if typ != nil || p.tok == token.ASSIGN || iota == 0 { + p.expect(token.ASSIGN) + values = p.parseRhsList() + } + p.expectSemi() // call before accessing p.linecomment + + // Go spec: The scope of a constant or variable identifier declared inside + // a function begins at the end of the ConstSpec or VarSpec and ends at + // the end of the innermost containing block. + // (Global identifiers are resolved in a separate phase after parsing.) + spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment} + p.declare(spec, p.topScope, ast.Con, idents...) + + return spec +} + + +func parseTypeSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec { + if p.trace { + defer un(trace(p, "TypeSpec")) + } + + ident := p.parseIdent() + + // Go spec: The scope of a type identifier declared inside a function begins + // at the identifier in the TypeSpec and ends at the end of the innermost + // containing block. + // (Global identifiers are resolved in a separate phase after parsing.) + spec := &ast.TypeSpec{doc, ident, nil, nil} + p.declare(spec, p.topScope, ast.Typ, ident) + + spec.Type = p.parseType() + p.expectSemi() // call before accessing p.linecomment + spec.Comment = p.lineComment + + return spec +} + + +func parseVarSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec { + if p.trace { + defer un(trace(p, "VarSpec")) + } + + idents := p.parseIdentList() + typ := p.tryType() + var values []ast.Expr + if typ == nil || p.tok == token.ASSIGN { + p.expect(token.ASSIGN) + values = p.parseRhsList() + } + p.expectSemi() // call before accessing p.linecomment + + // Go spec: The scope of a constant or variable identifier declared inside + // a function begins at the end of the ConstSpec or VarSpec and ends at + // the end of the innermost containing block. + // (Global identifiers are resolved in a separate phase after parsing.) + spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment} + p.declare(spec, p.topScope, ast.Var, idents...) + + return spec +} + + +func (p *parser) parseGenDecl(keyword token.Token, f parseSpecFunction) *ast.GenDecl { + if p.trace { + defer un(trace(p, "GenDecl("+keyword.String()+")")) + } + + doc := p.leadComment + pos := p.expect(keyword) + var lparen, rparen token.Pos + var list []ast.Spec + if p.tok == token.LPAREN { + lparen = p.pos + p.next() + for iota := 0; p.tok != token.RPAREN && p.tok != token.EOF; iota++ { + list = append(list, f(p, p.leadComment, iota)) + } + rparen = p.expect(token.RPAREN) + p.expectSemi() + } else { + list = append(list, f(p, nil, 0)) + } + + return &ast.GenDecl{doc, pos, keyword, lparen, list, rparen} +} + + +func (p *parser) parseReceiver(scope *ast.Scope) *ast.FieldList { + if p.trace { + defer un(trace(p, "Receiver")) + } + + pos := p.pos + par := p.parseParameters(scope, false) + + // must have exactly one receiver + if par.NumFields() != 1 { + p.errorExpected(pos, "exactly one receiver") + // TODO determine a better range for BadExpr below + par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{pos, pos}}} + return par + } + + // recv type must be of the form ["*"] identifier + recv := par.List[0] + base := deref(recv.Type) + if _, isIdent := base.(*ast.Ident); !isIdent { + p.errorExpected(base.Pos(), "(unqualified) identifier") + par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{recv.Pos(), recv.End()}}} + } + + return par +} + + +func (p *parser) parseFuncDecl() *ast.FuncDecl { + if p.trace { + defer un(trace(p, "FunctionDecl")) + } + + doc := p.leadComment + pos := p.expect(token.FUNC) + scope := ast.NewScope(p.topScope) // function scope + + var recv *ast.FieldList + if p.tok == token.LPAREN { + recv = p.parseReceiver(scope) + } + + ident := p.parseIdent() + + params, results := p.parseSignature(scope) + + var body *ast.BlockStmt + if p.tok == token.LBRACE { + body = p.parseBody(scope) + } + p.expectSemi() + + decl := &ast.FuncDecl{doc, recv, ident, &ast.FuncType{pos, params, results}, body} + if recv == nil { + // Go spec: The scope of an identifier denoting a constant, type, + // variable, or function (but not method) declared at top level + // (outside any function) is the package block. + // + // init() functions cannot be referred to and there may + // be more than one - don't put them in the pkgScope + if ident.Name != "init" { + p.declare(decl, p.pkgScope, ast.Fun, ident) + } + } + + return decl +} + + +func (p *parser) parseDecl() ast.Decl { + if p.trace { + defer un(trace(p, "Declaration")) + } + + var f parseSpecFunction + switch p.tok { + case token.CONST: + f = parseConstSpec + + case token.TYPE: + f = parseTypeSpec + + case token.VAR: + f = parseVarSpec + + case token.FUNC: + return p.parseFuncDecl() + + default: + pos := p.pos + p.errorExpected(pos, "declaration") + p.next() // make progress + decl := &ast.BadDecl{pos, p.pos} + return decl + } + + return p.parseGenDecl(p.tok, f) +} + + +func (p *parser) parseDeclList() (list []ast.Decl) { + if p.trace { + defer un(trace(p, "DeclList")) + } + + for p.tok != token.EOF { + list = append(list, p.parseDecl()) + } + + return +} + + +// ---------------------------------------------------------------------------- +// Source files + +func (p *parser) parseFile() *ast.File { + if p.trace { + defer un(trace(p, "File")) + } + + // package clause + doc := p.leadComment + pos := p.expect(token.PACKAGE) + // Go spec: The package clause is not a declaration; + // the package name does not appear in any scope. + ident := p.parseIdent() + if ident.Name == "_" { + p.error(p.pos, "invalid package name _") + } + p.expectSemi() + + var decls []ast.Decl + + // Don't bother parsing the rest if we had errors already. + // Likely not a Go source file at all. + + if p.ErrorCount() == 0 && p.mode&PackageClauseOnly == 0 { + // import decls + for p.tok == token.IMPORT { + decls = append(decls, p.parseGenDecl(token.IMPORT, parseImportSpec)) + } + + if p.mode&ImportsOnly == 0 { + // rest of package body + for p.tok != token.EOF { + decls = append(decls, p.parseDecl()) + } + } + } + + assert(p.topScope == p.pkgScope, "imbalanced scopes") + + // resolve global identifiers within the same file + i := 0 + for _, ident := range p.unresolved { + // i <= index for current ident + assert(ident.Obj == unresolved, "object already resolved") + ident.Obj = p.pkgScope.Lookup(ident.Name) // also removes unresolved sentinel + if ident.Obj == nil { + p.unresolved[i] = ident + i++ + } + } + + // TODO(gri): store p.imports in AST + return &ast.File{doc, pos, ident, decls, p.pkgScope, p.imports, p.unresolved[0:i], p.comments} +} diff --git a/src/pkg/go/scanner/scanner.go b/src/pkg/go/scanner/scanner.go index 07b7454c8..509abeca5 100644 --- a/src/pkg/go/scanner/scanner.go +++ b/src/pkg/go/scanner/scanner.go @@ -134,36 +134,6 @@ func (S *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode uint } -func charString(ch int) string { - var s string - switch ch { - case -1: - return `EOF` - case '\a': - s = `\a` - case '\b': - s = `\b` - case '\f': - s = `\f` - case '\n': - s = `\n` - case '\r': - s = `\r` - case '\t': - s = `\t` - case '\v': - s = `\v` - case '\\': - s = `\\` - case '\'': - s = `\'` - default: - s = string(ch) - } - return "'" + s + "' (U+" + strconv.Itob(ch, 16) + ")" -} - - func (S *Scanner) error(offs int, msg string) { if S.err != nil { S.err.Error(S.file.Position(S.file.Pos(offs)), msg) @@ -327,6 +297,10 @@ func (S *Scanner) scanNumber(seenDecimalPoint bool) token.Token { // hexadecimal int S.next() S.scanMantissa(16) + if S.offset-offs <= 2 { + // only scanned "0x" or "0X" + S.error(offs, "illegal hexadecimal number") + } } else { // octal int or float seenDecimalDigit := false @@ -700,7 +674,7 @@ scanAgain: tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR) default: if S.mode&AllowIllegalChars == 0 { - S.error(offs, "illegal character "+charString(ch)) + S.error(offs, "illegal character "+strconv.QuoteRune(ch)) } insertSemi = S.insertSemi // preserve insertSemi info } diff --git a/src/pkg/go/scanner/scanner_test.go b/src/pkg/go/scanner/scanner_test.go index 8afb00ee5..ee1e830a1 100644 --- a/src/pkg/go/scanner/scanner_test.go +++ b/src/pkg/go/scanner/scanner_test.go @@ -89,7 +89,7 @@ var tokens = [...]elt{ literal, }, - // Operators and delimitors + // Operators and delimiters {token.ADD, "+", operator}, {token.SUB, "-", operator}, {token.MUL, "*", operator}, @@ -650,7 +650,9 @@ var errors = []struct { pos int err string }{ - {`#`, token.ILLEGAL, 0, "illegal character '#' (U+23)"}, + {"\a", token.ILLEGAL, 0, "illegal character '\\a'"}, + {`#`, token.ILLEGAL, 0, "illegal character '#'"}, + {`…`, token.ILLEGAL, 0, "illegal character '…'"}, {`' '`, token.CHAR, 0, ""}, {`''`, token.CHAR, 0, "illegal character literal"}, {`'\8'`, token.CHAR, 2, "unknown escape sequence"}, @@ -670,6 +672,8 @@ var errors = []struct { {"078e0", token.FLOAT, 0, ""}, {"078", token.INT, 0, "illegal octal number"}, {"07800000009", token.INT, 0, "illegal octal number"}, + {"0x", token.INT, 0, "illegal hexadecimal number"}, + {"0X", token.INT, 0, "illegal hexadecimal number"}, {"\"abc\x00def\"", token.STRING, 4, "illegal character NUL"}, {"\"abc\x80def\"", token.STRING, 4, "illegal UTF-8 encoding"}, } diff --git a/src/pkg/go/token/position.go b/src/pkg/go/token/position.go index 809e53f0a..23a3cc00f 100644 --- a/src/pkg/go/token/position.go +++ b/src/pkg/go/token/position.go @@ -94,10 +94,14 @@ func searchFiles(a []*File, x int) int { func (s *FileSet) file(p Pos) *File { + if f := s.last; f != nil && f.base <= int(p) && int(p) <= f.base+f.size { + return f + } if i := searchFiles(s.files, int(p)); i >= 0 { f := s.files[i] // f.base <= int(p) by definition of searchFiles if int(p) <= f.base+f.size { + s.last = f return f } } @@ -131,7 +135,7 @@ func (f *File) position(p Pos) (pos Position) { func (s *FileSet) Position(p Pos) (pos Position) { if p != NoPos { // TODO(gri) consider optimizing the case where p - // is in the last file addded, or perhaps + // is in the last file added, or perhaps // looked at - will eliminate one level // of search s.mutex.RLock() @@ -316,8 +320,26 @@ func (f *File) Position(p Pos) (pos Position) { } -func searchUints(a []int, x int) int { - return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1 +func searchInts(a []int, x int) int { + // This function body is a manually inlined version of: + // + // return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1 + // + // With better compiler optimizations, this may not be needed in the + // future, but at the moment this change improves the go/printer + // benchmark performance by ~30%. This has a direct impact on the + // speed of gofmt and thus seems worthwhile (2011-04-29). + i, j := 0, len(a) + for i < j { + h := i + (j-i)/2 // avoid overflow when computing h + // i ≤ h < j + if a[h] <= x { + i = h + 1 + } else { + j = h + } + } + return i - 1 } @@ -329,14 +351,17 @@ func searchLineInfos(a []lineInfo, x int) int { // info returns the file name, line, and column number for a file offset. func (f *File) info(offset int) (filename string, line, column int) { filename = f.name - if i := searchUints(f.lines, offset); i >= 0 { + if i := searchInts(f.lines, offset); i >= 0 { line, column = i+1, offset-f.lines[i]+1 } - if i := searchLineInfos(f.infos, offset); i >= 0 { - alt := &f.infos[i] - filename = alt.filename - if i := searchUints(f.lines, alt.offset); i >= 0 { - line += alt.line - i - 1 + if len(f.infos) > 0 { + // almost no files have extra line infos + if i := searchLineInfos(f.infos, offset); i >= 0 { + alt := &f.infos[i] + filename = alt.filename + if i := searchInts(f.lines, alt.offset); i >= 0 { + line += alt.line - i - 1 + } } } return @@ -348,10 +373,10 @@ func (f *File) info(offset int) (filename string, line, column int) { // may invoke them concurrently. // type FileSet struct { - mutex sync.RWMutex // protects the file set - base int // base offset for the next file - files []*File // list of files in the order added to the set - index map[*File]int // file -> files index for quick lookup + mutex sync.RWMutex // protects the file set + base int // base offset for the next file + files []*File // list of files in the order added to the set + last *File // cache of last file looked up } @@ -359,7 +384,6 @@ type FileSet struct { func NewFileSet() *FileSet { s := new(FileSet) s.base = 1 // 0 == NoPos - s.index = make(map[*File]int) return s } @@ -405,8 +429,8 @@ func (s *FileSet) AddFile(filename string, base, size int) *File { } // add the file to the file set s.base = base - s.index[f] = len(s.files) s.files = append(s.files, f) + s.last = f return f } diff --git a/src/pkg/go/types/Makefile b/src/pkg/go/types/Makefile index 54e762b36..4ca707c73 100644 --- a/src/pkg/go/types/Makefile +++ b/src/pkg/go/types/Makefile @@ -6,6 +6,7 @@ include ../../../Make.inc TARG=go/types GOFILES=\ + check.go\ const.go\ exportdata.go\ gcimporter.go\ diff --git a/src/pkg/go/types/check.go b/src/pkg/go/types/check.go new file mode 100644 index 000000000..02d662926 --- /dev/null +++ b/src/pkg/go/types/check.go @@ -0,0 +1,233 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements the Check function, which typechecks a package. + +package types + +import ( + "fmt" + "go/ast" + "go/scanner" + "go/token" + "os" + "strconv" +) + + +const debug = false + + +type checker struct { + fset *token.FileSet + scanner.ErrorVector + types map[ast.Expr]Type +} + + +func (c *checker) errorf(pos token.Pos, format string, args ...interface{}) string { + msg := fmt.Sprintf(format, args...) + c.Error(c.fset.Position(pos), msg) + return msg +} + + +// collectFields collects struct fields tok = token.STRUCT), interface methods +// (tok = token.INTERFACE), and function arguments/results (tok = token.FUNC). +func (c *checker) collectFields(tok token.Token, list *ast.FieldList, cycleOk bool) (fields ObjList, tags []string, isVariadic bool) { + if list != nil { + for _, field := range list.List { + ftype := field.Type + if t, ok := ftype.(*ast.Ellipsis); ok { + ftype = t.Elt + isVariadic = true + } + typ := c.makeType(ftype, cycleOk) + tag := "" + if field.Tag != nil { + assert(field.Tag.Kind == token.STRING) + tag, _ = strconv.Unquote(field.Tag.Value) + } + if len(field.Names) > 0 { + // named fields + for _, name := range field.Names { + obj := name.Obj + obj.Type = typ + fields = append(fields, obj) + if tok == token.STRUCT { + tags = append(tags, tag) + } + } + } else { + // anonymous field + switch tok { + case token.STRUCT: + tags = append(tags, tag) + fallthrough + case token.FUNC: + obj := ast.NewObj(ast.Var, "") + obj.Type = typ + fields = append(fields, obj) + case token.INTERFACE: + utyp := Underlying(typ) + if typ, ok := utyp.(*Interface); ok { + // TODO(gri) This is not good enough. Check for double declarations! + fields = append(fields, typ.Methods...) + } else if _, ok := utyp.(*Bad); !ok { + // if utyp is Bad, don't complain (the root cause was reported before) + c.errorf(ftype.Pos(), "interface contains embedded non-interface type") + } + default: + panic("unreachable") + } + } + } + } + return +} + + +// makeType makes a new type for an AST type specification x or returns +// the type referred to by a type name x. If cycleOk is set, a type may +// refer to itself directly or indirectly; otherwise cycles are errors. +// +func (c *checker) makeType(x ast.Expr, cycleOk bool) (typ Type) { + if debug { + fmt.Printf("makeType (cycleOk = %v)\n", cycleOk) + ast.Print(c.fset, x) + defer func() { + fmt.Printf("-> %T %v\n\n", typ, typ) + }() + } + + switch t := x.(type) { + case *ast.BadExpr: + return &Bad{} + + case *ast.Ident: + // type name + obj := t.Obj + if obj == nil { + // unresolved identifier (error has been reported before) + return &Bad{Msg: "unresolved identifier"} + } + if obj.Kind != ast.Typ { + msg := c.errorf(t.Pos(), "%s is not a type", t.Name) + return &Bad{Msg: msg} + } + c.checkObj(obj, cycleOk) + if !cycleOk && obj.Type.(*Name).Underlying == nil { + // TODO(gri) Enable this message again once its position + // is independent of the underlying map implementation. + // msg := c.errorf(obj.Pos(), "illegal cycle in declaration of %s", obj.Name) + msg := "illegal cycle" + return &Bad{Msg: msg} + } + return obj.Type.(Type) + + case *ast.ParenExpr: + return c.makeType(t.X, cycleOk) + + case *ast.SelectorExpr: + // qualified identifier + // TODO (gri) eventually, this code belongs to expression + // type checking - here for the time being + if ident, ok := t.X.(*ast.Ident); ok { + if obj := ident.Obj; obj != nil { + if obj.Kind != ast.Pkg { + msg := c.errorf(ident.Pos(), "%s is not a package", obj.Name) + return &Bad{Msg: msg} + } + // TODO(gri) we have a package name but don't + // have the mapping from package name to package + // scope anymore (created in ast.NewPackage). + return &Bad{} // for now + } + } + // TODO(gri) can this really happen (the parser should have excluded this)? + msg := c.errorf(t.Pos(), "expected qualified identifier") + return &Bad{Msg: msg} + + case *ast.StarExpr: + return &Pointer{Base: c.makeType(t.X, true)} + + case *ast.ArrayType: + if t.Len != nil { + // TODO(gri) compute length + return &Array{Elt: c.makeType(t.Elt, cycleOk)} + } + return &Slice{Elt: c.makeType(t.Elt, true)} + + case *ast.StructType: + fields, tags, _ := c.collectFields(token.STRUCT, t.Fields, cycleOk) + return &Struct{Fields: fields, Tags: tags} + + case *ast.FuncType: + params, _, _ := c.collectFields(token.FUNC, t.Params, true) + results, _, isVariadic := c.collectFields(token.FUNC, t.Results, true) + return &Func{Recv: nil, Params: params, Results: results, IsVariadic: isVariadic} + + case *ast.InterfaceType: + methods, _, _ := c.collectFields(token.INTERFACE, t.Methods, cycleOk) + methods.Sort() + return &Interface{Methods: methods} + + case *ast.MapType: + return &Map{Key: c.makeType(t.Key, true), Elt: c.makeType(t.Key, true)} + + case *ast.ChanType: + return &Chan{Dir: t.Dir, Elt: c.makeType(t.Value, true)} + } + + panic(fmt.Sprintf("unreachable (%T)", x)) +} + + +// checkObj type checks an object. +func (c *checker) checkObj(obj *ast.Object, ref bool) { + if obj.Type != nil { + // object has already been type checked + return + } + + switch obj.Kind { + case ast.Bad: + // ignore + + case ast.Con: + // TODO(gri) complete this + + case ast.Typ: + typ := &Name{Obj: obj} + obj.Type = typ // "mark" object so recursion terminates + typ.Underlying = Underlying(c.makeType(obj.Decl.(*ast.TypeSpec).Type, ref)) + + case ast.Var: + // TODO(gri) complete this + + case ast.Fun: + // TODO(gri) complete this + + default: + panic("unreachable") + } +} + + +// Check typechecks a package. +// It augments the AST by assigning types to all ast.Objects and returns a map +// of types for all expression nodes in statements, and a scanner.ErrorList if +// there are errors. +// +func Check(fset *token.FileSet, pkg *ast.Package) (types map[ast.Expr]Type, err os.Error) { + var c checker + c.fset = fset + c.types = make(map[ast.Expr]Type) + + for _, obj := range pkg.Scope.Objects { + c.checkObj(obj, false) + } + + return c.types, c.GetError(scanner.NoMultiples) +} diff --git a/src/pkg/go/types/check_test.go b/src/pkg/go/types/check_test.go new file mode 100644 index 000000000..6ecb12b1e --- /dev/null +++ b/src/pkg/go/types/check_test.go @@ -0,0 +1,224 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements a typechecker test harness. The packages specified +// in tests are typechecked. Error messages reported by the typechecker are +// compared against the error messages expected in the test files. +// +// Expected errors are indicated in the test files by putting a comment +// of the form /* ERROR "rx" */ immediately following an offending token. +// The harness will verify that an error matching the regular expression +// rx is reported at that source position. Consecutive comments may be +// used to indicate multiple errors for the same token position. +// +// For instance, the following test file indicates that a "not declared" +// error should be reported for the undeclared variable x: +// +// package p +// func f() { +// _ = x /* ERROR "not declared" */ + 1 +// } + +package types + +import ( + "fmt" + "go/ast" + "go/parser" + "go/scanner" + "go/token" + "io/ioutil" + "os" + "regexp" + "testing" +) + + +// The test filenames do not end in .go so that they are invisible +// to gofmt since they contain comments that must not change their +// positions relative to surrounding tokens. + +var tests = []struct { + name string + files []string +}{ + {"test0", []string{"testdata/test0.src"}}, +} + + +var fset = token.NewFileSet() + + +// TODO(gri) This functionality should be in token.Fileset. +func getFile(filename string) *token.File { + for f := range fset.Files() { + if f.Name() == filename { + return f + } + } + return nil +} + + +// TODO(gri) This functionality should be in token.Fileset. +func getPos(filename string, offset int) token.Pos { + if f := getFile(filename); f != nil { + return f.Pos(offset) + } + return token.NoPos +} + + +// TODO(gri) Need to revisit parser interface. We should be able to use parser.ParseFiles +// or a similar function instead. +func parseFiles(t *testing.T, testname string, filenames []string) (map[string]*ast.File, os.Error) { + files := make(map[string]*ast.File) + var errors scanner.ErrorList + for _, filename := range filenames { + if _, exists := files[filename]; exists { + t.Fatalf("%s: duplicate file %s", testname, filename) + } + file, err := parser.ParseFile(fset, filename, nil, parser.DeclarationErrors) + if file == nil { + t.Fatalf("%s: could not parse file %s", testname, filename) + } + files[filename] = file + if err != nil { + // if the parser returns a non-scanner.ErrorList error + // the file couldn't be read in the first place and + // file == nil; in that case we shouldn't reach here + errors = append(errors, err.(scanner.ErrorList)...) + } + + } + return files, errors +} + + +// ERROR comments must be of the form /* ERROR "rx" */ and rx is +// a regular expression that matches the expected error message. +// +var errRx = regexp.MustCompile(`^/\* *ERROR *"([^"]*)" *\*/$`) + +// expectedErrors collects the regular expressions of ERROR comments found +// in files and returns them as a map of error positions to error messages. +// +func expectedErrors(t *testing.T, testname string, files map[string]*ast.File) map[token.Pos]string { + errors := make(map[token.Pos]string) + for filename := range files { + src, err := ioutil.ReadFile(filename) + if err != nil { + t.Fatalf("%s: could not read %s", testname, filename) + } + + var s scanner.Scanner + // file was parsed already - do not add it again to the file + // set otherwise the position information returned here will + // not match the position information collected by the parser + s.Init(getFile(filename), src, nil, scanner.ScanComments) + var prev token.Pos // position of last non-comment token + + scanFile: + for { + pos, tok, lit := s.Scan() + switch tok { + case token.EOF: + break scanFile + case token.COMMENT: + s := errRx.FindStringSubmatch(lit) + if len(s) == 2 { + errors[prev] = string(s[1]) + } + default: + prev = pos + } + } + } + return errors +} + + +func eliminate(t *testing.T, expected map[token.Pos]string, errors os.Error) { + if errors == nil { + return + } + for _, error := range errors.(scanner.ErrorList) { + // error.Pos is a token.Position, but we want + // a token.Pos so we can do a map lookup + // TODO(gri) Need to move scanner.Errors over + // to use token.Pos and file set info. + pos := getPos(error.Pos.Filename, error.Pos.Offset) + if msg, found := expected[pos]; found { + // we expect a message at pos; check if it matches + rx, err := regexp.Compile(msg) + if err != nil { + t.Errorf("%s: %v", error.Pos, err) + continue + } + if match := rx.MatchString(error.Msg); !match { + t.Errorf("%s: %q does not match %q", error.Pos, error.Msg, msg) + continue + } + // we have a match - eliminate this error + expected[pos] = "", false + } else { + // To keep in mind when analyzing failed test output: + // If the same error position occurs multiple times in errors, + // this message will be triggered (because the first error at + // the position removes this position from the expected errors). + t.Errorf("%s: no (multiple?) error expected, but found: %s", error.Pos, error.Msg) + } + } +} + + +func check(t *testing.T, testname string, testfiles []string) { + // TODO(gri) Eventually all these different phases should be + // subsumed into a single function call that takes + // a set of files and creates a fully resolved and + // type-checked AST. + + files, err := parseFiles(t, testname, testfiles) + + // we are expecting the following errors + // (collect these after parsing the files so that + // they are found in the file set) + errors := expectedErrors(t, testname, files) + + // verify errors returned by the parser + eliminate(t, errors, err) + + // verify errors returned after resolving identifiers + pkg, err := ast.NewPackage(fset, files, GcImporter, Universe) + eliminate(t, errors, err) + + // verify errors returned by the typechecker + _, err = Check(fset, pkg) + eliminate(t, errors, err) + + // there should be no expected errors left + if len(errors) > 0 { + t.Errorf("%s: %d errors not reported:", testname, len(errors)) + for pos, msg := range errors { + t.Errorf("%s: %s\n", fset.Position(pos), msg) + } + } +} + + +func TestCheck(t *testing.T) { + // For easy debugging w/o changing the testing code, + // if there is a local test file, only test that file. + const testfile = "test.go" + if fi, err := os.Stat(testfile); err == nil && fi.IsRegular() { + fmt.Printf("WARNING: Testing only %s (remove it to run all tests)\n", testfile) + check(t, testfile, []string{testfile}) + return + } + + // Otherwise, run all the tests. + for _, test := range tests { + check(t, test.name, test.files) + } +} diff --git a/src/pkg/go/types/gcimporter.go b/src/pkg/go/types/gcimporter.go index 30adc04e7..2cfed7726 100644 --- a/src/pkg/go/types/gcimporter.go +++ b/src/pkg/go/types/gcimporter.go @@ -74,15 +74,14 @@ func findPkg(path string) (filename, id string) { // object/archive file and populates its scope with the results. type gcParser struct { scanner scanner.Scanner - tok int // current token - lit string // literal string; only valid for Ident, Int, String tokens - id string // package id of imported package - scope *ast.Scope // scope of imported package; alias for deps[id] - deps map[string]*ast.Scope // package id -> package scope + tok int // current token + lit string // literal string; only valid for Ident, Int, String tokens + id string // package id of imported package + imports map[string]*ast.Object // package id -> package object } -func (p *gcParser) init(filename, id string, src io.Reader) { +func (p *gcParser) init(filename, id string, src io.Reader, imports map[string]*ast.Object) { p.scanner.Init(src) p.scanner.Error = func(_ *scanner.Scanner, msg string) { p.error(msg) } p.scanner.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings | scanner.ScanComments | scanner.SkipComments @@ -90,8 +89,7 @@ func (p *gcParser) init(filename, id string, src io.Reader) { p.scanner.Filename = filename // for good error messages p.next() p.id = id - p.scope = ast.NewScope(nil) - p.deps = map[string]*ast.Scope{"unsafe": Unsafe, id: p.scope} + p.imports = imports } @@ -110,9 +108,9 @@ func (p *gcParser) next() { // GcImporter implements the ast.Importer signature. -func GcImporter(path string) (name string, scope *ast.Scope, err os.Error) { +func GcImporter(imports map[string]*ast.Object, path string) (pkg *ast.Object, err os.Error) { if path == "unsafe" { - return path, Unsafe, nil + return Unsafe, nil } defer func() { @@ -130,6 +128,10 @@ func GcImporter(path string) (name string, scope *ast.Scope, err os.Error) { return } + if pkg = imports[id]; pkg != nil { + return // package was imported before + } + buf, err := ExportData(filename) if err != nil { return @@ -137,13 +139,12 @@ func GcImporter(path string) (name string, scope *ast.Scope, err os.Error) { defer buf.Close() if trace { - fmt.Printf("importing %s\n", filename) + fmt.Printf("importing %s (%s)\n", id, filename) } var p gcParser - p.init(filename, id, buf) - name, scope = p.parseExport() - + p.init(filename, id, buf, imports) + pkg = p.parseExport() return } @@ -214,21 +215,31 @@ func (p *gcParser) expectKeyword(keyword string) { // ImportPath = string_lit . // -func (p *gcParser) parsePkgId() *ast.Scope { +func (p *gcParser) parsePkgId() *ast.Object { id, err := strconv.Unquote(p.expect(scanner.String)) if err != nil { p.error(err) } - scope := p.scope // id == "" stands for the imported package id - if id != "" { - if scope = p.deps[id]; scope == nil { - scope = ast.NewScope(nil) - p.deps[id] = scope - } + switch id { + case "": + // id == "" stands for the imported package id + // (only known at time of package installation) + id = p.id + case "unsafe": + // package unsafe is not in the imports map - handle explicitly + return Unsafe + } + + pkg := p.imports[id] + if pkg == nil { + scope = ast.NewScope(nil) + pkg = ast.NewObj(ast.Pkg, "") + pkg.Data = scope + p.imports[id] = pkg } - return scope + return pkg } @@ -253,13 +264,14 @@ func (p *gcParser) parseDotIdent() string { // ExportedName = ImportPath "." dotIdentifier . // func (p *gcParser) parseExportedName(kind ast.ObjKind) *ast.Object { - scope := p.parsePkgId() + pkg := p.parsePkgId() p.expect('.') name := p.parseDotIdent() // a type may have been declared before - if it exists // already in the respective package scope, return that // type + scope := pkg.Data.(*ast.Scope) if kind == ast.Typ { if obj := scope.Lookup(name); obj != nil { assert(obj.Kind == ast.Typ) @@ -344,28 +356,22 @@ func (p *gcParser) parseName() (name string) { // Field = Name Type [ ":" string_lit ] . // -func (p *gcParser) parseField(scope *ast.Scope) { - // TODO(gri) The code below is not correct for anonymous fields: - // The name is the type name; it should not be empty. +func (p *gcParser) parseField() (fld *ast.Object, tag string) { name := p.parseName() ftyp := p.parseType() if name == "" { // anonymous field - ftyp must be T or *T and T must be a type name - ftyp = Deref(ftyp) - if ftyp, ok := ftyp.(*Name); ok { - name = ftyp.Obj.Name - } else { + if _, ok := Deref(ftyp).(*Name); !ok { p.errorf("anonymous field expected") } } if p.tok == ':' { p.next() - tag := p.expect(scanner.String) - _ = tag // TODO(gri) store tag somewhere + tag = p.expect(scanner.String) } - fld := ast.NewObj(ast.Var, name) + fld = ast.NewObj(ast.Var, name) fld.Type = ftyp - scope.Insert(fld) + return } @@ -373,103 +379,124 @@ func (p *gcParser) parseField(scope *ast.Scope) { // FieldList = Field { ";" Field } . // func (p *gcParser) parseStructType() Type { + var fields []*ast.Object + var tags []string + + parseField := func() { + fld, tag := p.parseField() + fields = append(fields, fld) + tags = append(tags, tag) + } + p.expectKeyword("struct") p.expect('{') - scope := ast.NewScope(nil) if p.tok != '}' { - p.parseField(scope) + parseField() for p.tok == ';' { p.next() - p.parseField(scope) + parseField() } } p.expect('}') - return &Struct{} + + return &Struct{Fields: fields, Tags: tags} } -// Parameter = ( identifier | "?" ) [ "..." ] Type . +// Parameter = ( identifier | "?" ) [ "..." ] Type [ ":" string_lit ] . // -func (p *gcParser) parseParameter(scope *ast.Scope, isVariadic *bool) { +func (p *gcParser) parseParameter() (par *ast.Object, isVariadic bool) { name := p.parseName() if name == "" { name = "_" // cannot access unnamed identifiers } - if isVariadic != nil { - if *isVariadic { - p.error("... not on final argument") - } - if p.tok == '.' { - p.expectSpecial("...") - *isVariadic = true - } + if p.tok == '.' { + p.expectSpecial("...") + isVariadic = true } ptyp := p.parseType() - par := ast.NewObj(ast.Var, name) + // ignore argument tag + if p.tok == ':' { + p.next() + p.expect(scanner.String) + } + par = ast.NewObj(ast.Var, name) par.Type = ptyp - scope.Insert(par) + return } // Parameters = "(" [ ParameterList ] ")" . // ParameterList = { Parameter "," } Parameter . // -func (p *gcParser) parseParameters(scope *ast.Scope, isVariadic *bool) { +func (p *gcParser) parseParameters() (list []*ast.Object, isVariadic bool) { + parseParameter := func() { + par, variadic := p.parseParameter() + list = append(list, par) + if variadic { + if isVariadic { + p.error("... not on final argument") + } + isVariadic = true + } + } + p.expect('(') if p.tok != ')' { - p.parseParameter(scope, isVariadic) + parseParameter() for p.tok == ',' { p.next() - p.parseParameter(scope, isVariadic) + parseParameter() } } p.expect(')') + + return } // Signature = Parameters [ Result ] . // Result = Type | Parameters . // -func (p *gcParser) parseSignature(scope *ast.Scope, isVariadic *bool) { - p.parseParameters(scope, isVariadic) +func (p *gcParser) parseSignature() *Func { + params, isVariadic := p.parseParameters() // optional result type + var results []*ast.Object switch p.tok { case scanner.Ident, scanner.String, '[', '*', '<': // single, unnamed result result := ast.NewObj(ast.Var, "_") result.Type = p.parseType() - scope.Insert(result) + results = []*ast.Object{result} case '(': // named or multiple result(s) - p.parseParameters(scope, nil) + var variadic bool + results, variadic = p.parseParameters() + if variadic { + p.error("... not permitted on result type") + } } -} - -// FuncType = "func" Signature . -// -func (p *gcParser) parseFuncType() Type { - // "func" already consumed - scope := ast.NewScope(nil) - isVariadic := false - p.parseSignature(scope, &isVariadic) - return &Func{IsVariadic: isVariadic} + return &Func{Params: params, Results: results, IsVariadic: isVariadic} } // MethodSpec = identifier Signature . // -func (p *gcParser) parseMethodSpec(scope *ast.Scope) { +func (p *gcParser) parseMethodSpec() *ast.Object { if p.tok == scanner.Ident { p.expect(scanner.Ident) } else { + // TODO(gri) should this be parseExportedName here? p.parsePkgId() p.expect('.') p.parseDotIdent() } - isVariadic := false - p.parseSignature(scope, &isVariadic) + p.parseSignature() + + // TODO(gri) compute method object + return ast.NewObj(ast.Fun, "_") } @@ -477,18 +504,26 @@ func (p *gcParser) parseMethodSpec(scope *ast.Scope) { // MethodList = MethodSpec { ";" MethodSpec } . // func (p *gcParser) parseInterfaceType() Type { + var methods ObjList + + parseMethod := func() { + meth := p.parseMethodSpec() + methods = append(methods, meth) + } + p.expectKeyword("interface") p.expect('{') - scope := ast.NewScope(nil) if p.tok != '}' { - p.parseMethodSpec(scope) + parseMethod() for p.tok == ';' { p.next() - p.parseMethodSpec(scope) + parseMethod() } } p.expect('}') - return &Interface{} + + methods.Sort() + return &Interface{Methods: methods} } @@ -520,6 +555,7 @@ func (p *gcParser) parseChanType() Type { // TypeName = ExportedName . // SliceType = "[" "]" Type . // PointerType = "*" Type . +// FuncType = "func" Signature . // func (p *gcParser) parseType() Type { switch p.tok { @@ -530,8 +566,9 @@ func (p *gcParser) parseType() Type { case "struct": return p.parseStructType() case "func": - p.next() // parseFuncType assumes "func" is already consumed - return p.parseFuncType() + // FuncType + p.next() + return p.parseSignature() case "interface": return p.parseInterfaceType() case "map": @@ -578,9 +615,10 @@ func (p *gcParser) parseImportDecl() { // The identifier has no semantic meaning in the import data. // It exists so that error messages can print the real package // name: binary.ByteOrder instead of "encoding/binary".ByteOrder. - // TODO(gri): Save package id -> package name mapping. - p.expect(scanner.Ident) - p.parsePkgId() + name := p.expect(scanner.Ident) + pkg := p.parsePkgId() + assert(pkg.Name == "" || pkg.Name == name) + pkg.Name = name } @@ -681,7 +719,7 @@ func (p *gcParser) parseConstDecl() { if obj.Type == nil { obj.Type = typ } - _ = x // TODO(gri) store x somewhere + obj.Data = x } @@ -690,12 +728,18 @@ func (p *gcParser) parseConstDecl() { func (p *gcParser) parseTypeDecl() { p.expectKeyword("type") obj := p.parseExportedName(ast.Typ) + + // The type object may have been imported before and thus already + // have a type associated with it. We still need to parse the type + // structure, but throw it away if the object already has a type. + // This ensures that all imports refer to the same type object for + // a given type declaration. typ := p.parseType() - name := obj.Type.(*Name) - assert(name.Underlying == nil) - assert(Underlying(typ) == typ) - name.Underlying = typ + if name := obj.Type.(*Name); name.Underlying == nil { + assert(Underlying(typ) == typ) + name.Underlying = typ + } } @@ -713,7 +757,7 @@ func (p *gcParser) parseVarDecl() { func (p *gcParser) parseFuncDecl() { // "func" already consumed obj := p.parseExportedName(ast.Fun) - obj.Type = p.parseFuncType() + obj.Type = p.parseSignature() } @@ -722,14 +766,11 @@ func (p *gcParser) parseFuncDecl() { // func (p *gcParser) parseMethodDecl() { // "func" already consumed - scope := ast.NewScope(nil) // method scope p.expect('(') - p.parseParameter(scope, nil) // receiver + p.parseParameter() // receiver p.expect(')') p.expect(scanner.Ident) - isVariadic := false - p.parseSignature(scope, &isVariadic) - + p.parseSignature() } @@ -763,7 +804,7 @@ func (p *gcParser) parseDecl() { // Export = "PackageClause { Decl } "$$" . // PackageClause = "package" identifier [ "safe" ] "\n" . // -func (p *gcParser) parseExport() (string, *ast.Scope) { +func (p *gcParser) parseExport() *ast.Object { p.expectKeyword("package") name := p.expect(scanner.Ident) if p.tok != '\n' { @@ -774,6 +815,11 @@ func (p *gcParser) parseExport() (string, *ast.Scope) { } p.expect('\n') + assert(p.imports[p.id] == nil) + pkg := ast.NewObj(ast.Pkg, name) + pkg.Data = ast.NewScope(nil) + p.imports[p.id] = pkg + for p.tok != '$' && p.tok != scanner.EOF { p.parseDecl() } @@ -788,5 +834,5 @@ func (p *gcParser) parseExport() (string, *ast.Scope) { p.errorf("expected no scanner errors, got %d", n) } - return name, p.scope + return pkg } diff --git a/src/pkg/go/types/gcimporter_test.go b/src/pkg/go/types/gcimporter_test.go index 556e761df..10240add5 100644 --- a/src/pkg/go/types/gcimporter_test.go +++ b/src/pkg/go/types/gcimporter_test.go @@ -6,6 +6,7 @@ package types import ( "exec" + "go/ast" "io/ioutil" "path/filepath" "runtime" @@ -36,29 +37,23 @@ func init() { func compile(t *testing.T, dirname, filename string) { - cmd, err := exec.Run(gcPath, []string{gcPath, filename}, nil, dirname, exec.DevNull, exec.Pipe, exec.MergeWithStdout) + cmd := exec.Command(gcPath, filename) + cmd.Dir = dirname + out, err := cmd.CombinedOutput() if err != nil { t.Errorf("%s %s failed: %s", gcName, filename, err) return } - defer cmd.Close() - - msg, err := cmd.Wait(0) - if err != nil { - t.Errorf("%s %s failed: %s", gcName, filename, err) - return - } - - if !msg.Exited() || msg.ExitStatus() != 0 { - t.Errorf("%s %s failed: exit status = %d", gcName, filename, msg.ExitStatus()) - output, _ := ioutil.ReadAll(cmd.Stdout) - t.Log(string(output)) - } + t.Logf("%s", string(out)) } +// Use the same global imports map for all tests. The effect is +// as if all tested packages were imported into a single package. +var imports = make(map[string]*ast.Object) + func testPath(t *testing.T, path string) bool { - _, _, err := GcImporter(path) + _, err := GcImporter(imports, path) if err != nil { t.Errorf("testPath(%s): %s", path, err) return false diff --git a/src/pkg/go/types/testdata/exports.go b/src/pkg/go/types/testdata/exports.go index 13efe012a..1de2e00ad 100644 --- a/src/pkg/go/types/testdata/exports.go +++ b/src/pkg/go/types/testdata/exports.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// This file is used to generate a .6 object file which +// This file is used to generate an object file which // serves as test file for gcimporter_test.go. package exports @@ -14,13 +14,13 @@ import ( const ( C0 int = 0 - C1 = 3.14159265 - C2 = 2.718281828i - C3 = -123.456e-789 - C4 = +123.456E+789 - C5 = 1234i - C6 = "foo\n" - C7 = `bar\n` + C1 = 3.14159265 + C2 = 2.718281828i + C3 = -123.456e-789 + C4 = +123.456E+789 + C5 = 1234i + C6 = "foo\n" + C7 = `bar\n` ) diff --git a/src/pkg/go/types/testdata/test0.src b/src/pkg/go/types/testdata/test0.src new file mode 100644 index 000000000..84a1abe27 --- /dev/null +++ b/src/pkg/go/types/testdata/test0.src @@ -0,0 +1,154 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// type declarations + +package test0 + +import "unsafe" + +const pi = 3.1415 + +type ( + N undeclared /* ERROR "undeclared" */ + B bool + I int32 + A [10]P + T struct { + x, y P + } + P *T + R (*R) + F func(A) I + Y interface { + f(A) I + } + S [](((P))) + M map[I]F + C chan<- I +) + + +type ( + p1 pi /* ERROR "not a package" */ .foo + p2 unsafe.Pointer +) + + +type ( + Pi pi /* ERROR "not a type" */ + + a /* DISABLED "illegal cycle" */ a + a /* ERROR "redeclared" */ int + + // where the cycle error appears depends on the + // order in which declarations are processed + // (which depends on the order in which a map + // is iterated through) + b c + c /* DISABLED "illegal cycle" */ d + d e + e b + + t *t + + U V + V *W + W U + + P1 *S2 + P2 P1 + + S0 struct { + } + S1 struct { + a, b, c int + u, v, a /* ERROR "redeclared" */ float32 + } + S2 struct { + U // anonymous field + // TODO(gri) recognize double-declaration below + // U /* ERROR "redeclared" */ int + } + S3 struct { + x S2 + } + S4/* DISABLED "illegal cycle" */ struct { + S4 + } + S5 struct { + S6 + } + S6 /* DISABLED "illegal cycle" */ struct { + field S7 + } + S7 struct { + S5 + } + + L1 []L1 + L2 []int + + A1 [10]int + A2 /* DISABLED "illegal cycle" */ [10]A2 + A3 /* DISABLED "illegal cycle" */ [10]struct { + x A4 + } + A4 [10]A3 + + F1 func() + F2 func(x, y, z float32) + F3 func(x, y, x /* ERROR "redeclared" */ float32) + F4 func() (x, y, x /* ERROR "redeclared" */ float32) + F5 func(x int) (x /* ERROR "redeclared" */ float32) + F6 func(x ...int) + + I1 interface{} + I2 interface { + m1() + } + I3 interface { + m1() + m1 /* ERROR "redeclared" */ () + } + I4 interface { + m1(x, y, x /* ERROR "redeclared" */ float32) + m2() (x, y, x /* ERROR "redeclared" */ float32) + m3(x int) (x /* ERROR "redeclared" */ float32) + } + I5 interface { + m1(I5) + } + I6 interface { + S0 /* ERROR "non-interface" */ + } + I7 interface { + I1 + I1 + } + I8 /* DISABLED "illegal cycle" */ interface { + I8 + } + I9 /* DISABLED "illegal cycle" */ interface { + I10 + } + I10 interface { + I11 + } + I11 interface { + I9 + } + + C1 chan int + C2 <-chan int + C3 chan<- C3 + C4 chan C5 + C5 chan C6 + C6 chan C4 + + M1 map[Last]string + M2 map[string]M2 + + Last int +) diff --git a/src/pkg/go/types/types.go b/src/pkg/go/types/types.go index 2ee645d98..10b0145b8 100644 --- a/src/pkg/go/types/types.go +++ b/src/pkg/go/types/types.go @@ -7,7 +7,10 @@ // package types -import "go/ast" +import ( + "go/ast" + "sort" +) // All types implement the Type interface. @@ -23,6 +26,13 @@ type ImplementsType struct{} func (t *ImplementsType) isType() {} +// A Bad type is a non-nil placeholder type when we don't know a type. +type Bad struct { + ImplementsType + Msg string // for better error reporting/debugging +} + + // A Basic represents a (unnamed) basic type. type Basic struct { ImplementsType @@ -46,9 +56,15 @@ type Slice struct { // A Struct represents a struct type struct{...}. +// Anonymous fields are represented by objects with empty names. type Struct struct { ImplementsType - // TODO(gri) need to remember fields. + Fields ObjList // struct fields; or nil + Tags []string // corresponding tags; or nil + // TODO(gri) This type needs some rethinking: + // - at the moment anonymous fields are marked with "" object names, + // and their names have to be reconstructed + // - there is no scope for fast lookup (but the parser creates one) } @@ -60,17 +76,20 @@ type Pointer struct { // A Func represents a function type func(...) (...). +// Unnamed parameters are represented by objects with empty names. type Func struct { ImplementsType - IsVariadic bool - // TODO(gri) need to remember parameters. + Recv *ast.Object // nil if not a method + Params ObjList // (incoming) parameters from left to right; or nil + Results ObjList // (outgoing) results from left to right; or nil + IsVariadic bool // true if the last parameter's type is of the form ...T } // An Interface represents an interface type interface{...}. type Interface struct { ImplementsType - // TODO(gri) need to remember methods. + Methods ObjList // interface methods sorted by name; or nil } @@ -112,11 +131,143 @@ func Deref(typ Type) Type { func Underlying(typ Type) Type { if typ, ok := typ.(*Name); ok { utyp := typ.Underlying - if _, ok := utyp.(*Basic); ok { - return typ + if _, ok := utyp.(*Basic); !ok { + return utyp } - return utyp - + // the underlying type of a type name referring + // to an (untyped) basic type is the basic type + // name } return typ } + + +// An ObjList represents an ordered (in some fashion) list of objects. +type ObjList []*ast.Object + +// ObjList implements sort.Interface. +func (list ObjList) Len() int { return len(list) } +func (list ObjList) Less(i, j int) bool { return list[i].Name < list[j].Name } +func (list ObjList) Swap(i, j int) { list[i], list[j] = list[j], list[i] } + +// Sort sorts an object list by object name. +func (list ObjList) Sort() { sort.Sort(list) } + + +// identicalTypes returns true if both lists a and b have the +// same length and corresponding objects have identical types. +func identicalTypes(a, b ObjList) bool { + if len(a) == len(b) { + for i, x := range a { + y := b[i] + if !Identical(x.Type.(Type), y.Type.(Type)) { + return false + } + } + return true + } + return false +} + + +// Identical returns true if two types are identical. +func Identical(x, y Type) bool { + if x == y { + return true + } + + switch x := x.(type) { + case *Bad: + // A Bad type is always identical to any other type + // (to avoid spurious follow-up errors). + return true + + case *Basic: + if y, ok := y.(*Basic); ok { + panic("unimplemented") + _ = y + } + + case *Array: + // Two array types are identical if they have identical element types + // and the same array length. + if y, ok := y.(*Array); ok { + return x.Len == y.Len && Identical(x.Elt, y.Elt) + } + + case *Slice: + // Two slice types are identical if they have identical element types. + if y, ok := y.(*Slice); ok { + return Identical(x.Elt, y.Elt) + } + + case *Struct: + // Two struct types are identical if they have the same sequence of fields, + // and if corresponding fields have the same names, and identical types, + // and identical tags. Two anonymous fields are considered to have the same + // name. Lower-case field names from different packages are always different. + if y, ok := y.(*Struct); ok { + // TODO(gri) handle structs from different packages + if identicalTypes(x.Fields, y.Fields) { + for i, f := range x.Fields { + g := y.Fields[i] + if f.Name != g.Name || x.Tags[i] != y.Tags[i] { + return false + } + } + return true + } + } + + case *Pointer: + // Two pointer types are identical if they have identical base types. + if y, ok := y.(*Pointer); ok { + return Identical(x.Base, y.Base) + } + + case *Func: + // Two function types are identical if they have the same number of parameters + // and result values, corresponding parameter and result types are identical, + // and either both functions are variadic or neither is. Parameter and result + // names are not required to match. + if y, ok := y.(*Func); ok { + return identicalTypes(x.Params, y.Params) && + identicalTypes(x.Results, y.Results) && + x.IsVariadic == y.IsVariadic + } + + case *Interface: + // Two interface types are identical if they have the same set of methods with + // the same names and identical function types. Lower-case method names from + // different packages are always different. The order of the methods is irrelevant. + if y, ok := y.(*Interface); ok { + return identicalTypes(x.Methods, y.Methods) // methods are sorted + } + + case *Map: + // Two map types are identical if they have identical key and value types. + if y, ok := y.(*Map); ok { + return Identical(x.Key, y.Key) && Identical(x.Elt, y.Elt) + } + + case *Chan: + // Two channel types are identical if they have identical value types + // and the same direction. + if y, ok := y.(*Chan); ok { + return x.Dir == y.Dir && Identical(x.Elt, y.Elt) + } + + case *Name: + // Two named types are identical if their type names originate + // in the same type declaration. + if y, ok := y.(*Name); ok { + return x.Obj == y.Obj || + // permit bad objects to be equal to avoid + // follow up errors + x.Obj != nil && x.Obj.Kind == ast.Bad || + y.Obj != nil && y.Obj.Kind == ast.Bad + } + } + + return false +} diff --git a/src/pkg/go/types/universe.go b/src/pkg/go/types/universe.go index 2a54a8ac1..96005cff5 100644 --- a/src/pkg/go/types/universe.go +++ b/src/pkg/go/types/universe.go @@ -11,9 +11,9 @@ import "go/ast" var ( - scope, // current scope to use for initialization - Universe, - Unsafe *ast.Scope + scope *ast.Scope // current scope to use for initialization + Universe *ast.Scope + Unsafe *ast.Object // package unsafe ) @@ -56,8 +56,8 @@ var ( func init() { - Universe = ast.NewScope(nil) - scope = Universe + scope = ast.NewScope(nil) + Universe = scope Bool = defType("bool") defType("byte") // TODO(gri) should be an alias for uint8 @@ -98,8 +98,10 @@ func init() { defFun("real") defFun("recover") - Unsafe = ast.NewScope(nil) - scope = Unsafe + scope = ast.NewScope(nil) + Unsafe = ast.NewObj(ast.Pkg, "unsafe") + Unsafe.Data = scope + defType("Pointer") defFun("Alignof") diff --git a/src/pkg/gob/decode.go b/src/pkg/gob/decode.go index 0e86df6b5..381d44c05 100644 --- a/src/pkg/gob/decode.go +++ b/src/pkg/gob/decode.go @@ -172,7 +172,7 @@ func ignoreTwoUints(i *decInstr, state *decoderState, p unsafe.Pointer) { state.decodeUint() } -// decBool decodes a uiint and stores it as a boolean through p. +// decBool decodes a uint and stores it as a boolean through p. func decBool(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { diff --git a/src/pkg/gob/decoder.go b/src/pkg/gob/decoder.go index ea2f62ec5..b83904a71 100644 --- a/src/pkg/gob/decoder.go +++ b/src/pkg/gob/decoder.go @@ -155,8 +155,8 @@ func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId { // Decode reads the next value from the connection and stores // it in the data represented by the empty interface value. // If e is nil, the value will be discarded. Otherwise, -// the value underlying e must either be the correct type for the next -// data item received, and must be a pointer. +// the value underlying e must be a pointer to the +// correct type for the next data item received. func (dec *Decoder) Decode(e interface{}) os.Error { if e == nil { return dec.DecodeValue(reflect.Value{}) diff --git a/src/pkg/gob/doc.go b/src/pkg/gob/doc.go index 189086f52..850759bbd 100644 --- a/src/pkg/gob/doc.go +++ b/src/pkg/gob/doc.go @@ -159,7 +159,7 @@ description, constructed from these types: Elem typeId Len int } - type CommonType { + type CommonType struct { Name string // the name of the struct type Id int // the id of the type, repeated so it's inside the type } diff --git a/src/pkg/gob/gobencdec_test.go b/src/pkg/gob/gobencdec_test.go index e94534f4c..3e1906020 100644 --- a/src/pkg/gob/gobencdec_test.go +++ b/src/pkg/gob/gobencdec_test.go @@ -384,7 +384,7 @@ func TestGobEncoderFieldTypeError(t *testing.T) { y := &GobTest1{} err = dec.Decode(y) if err == nil { - t.Fatal("expected decode error for mistmatched fields (non-encoder to decoder)") + t.Fatal("expected decode error for mismatched fields (non-encoder to decoder)") } if strings.Index(err.String(), "type") < 0 { t.Fatal("expected type error; got", err) diff --git a/src/pkg/gob/type.go b/src/pkg/gob/type.go index c5b8fb5d9..c6542633a 100644 --- a/src/pkg/gob/type.go +++ b/src/pkg/gob/type.go @@ -673,7 +673,7 @@ func mustGetTypeInfo(rt reflect.Type) *typeInfo { // A type that implements GobEncoder and GobDecoder has complete // control over the representation of its data and may therefore // contain things such as private fields, channels, and functions, -// which are not usually transmissable in gob streams. +// which are not usually transmissible in gob streams. // // Note: Since gobs can be stored permanently, It is good design // to guarantee the encoding used by a GobEncoder is stable as the diff --git a/src/pkg/html/doc.go b/src/pkg/html/doc.go index 55135c3d0..5bc063086 100644 --- a/src/pkg/html/doc.go +++ b/src/pkg/html/doc.go @@ -4,6 +4,7 @@ /* Package html implements an HTML5-compliant tokenizer and parser. +INCOMPLETE. Tokenization is done by creating a Tokenizer for an io.Reader r. It is the caller's responsibility to ensure that r provides UTF-8 encoded HTML. diff --git a/src/pkg/html/token.go b/src/pkg/html/token.go index ad03241ed..23c95ece6 100644 --- a/src/pkg/html/token.go +++ b/src/pkg/html/token.go @@ -331,10 +331,10 @@ func (z *Tokenizer) trim(i int) int { return k } -// lower finds the largest alphabetic [0-9A-Za-z]* word at the start of z.buf[i:] -// and returns that word lower-cased, as well as the trimmed cursor location -// after that word. -func (z *Tokenizer) lower(i int) ([]byte, int) { +// word finds the largest alphabetic [0-9A-Za-z]* word at the start +// of z.buf[i:] and returns that word (optionally lower-cased), as +// well as the trimmed cursor location after that word. +func (z *Tokenizer) word(i int, lower bool) ([]byte, int) { i0 := i loop: for ; i < z.p1; i++ { @@ -343,7 +343,9 @@ loop: case '0' <= c && c <= '9': // No-op. case 'A' <= c && c <= 'Z': - z.buf[i] = c + 'a' - 'A' + if lower { + z.buf[i] = c + 'a' - 'A' + } case 'a' <= c && c <= 'z': // No-op. default: @@ -353,6 +355,33 @@ loop: return z.buf[i0:i], z.trim(i) } +// attrName finds the largest attribute name at the start +// of z.buf[i:] and returns it lower-cased, as well +// as the trimmed cursor location after that word. +// +// http://dev.w3.org/html5/spec/Overview.html#syntax-attribute-name +// TODO: unicode characters +func (z *Tokenizer) attrName(i int) ([]byte, int) { + i0 := i +loop: + for ; i < z.p1; i++ { + c := z.buf[i] + switch c { + case '<', '>', '"', '\'', '/', '=': + break loop + } + switch { + case 'A' <= c && c <= 'Z': + z.buf[i] = c + 'a' - 'A' + case c > ' ' && c < 0x7f: + // No-op. + default: + break loop + } + } + return z.buf[i0:i], z.trim(i) +} + // Text returns the unescaped text of a TextToken or a CommentToken. // The contents of the returned slice may change on the next call to Next. func (z *Tokenizer) Text() []byte { @@ -388,7 +417,7 @@ func (z *Tokenizer) TagName() (name []byte, hasAttr bool) { if z.buf[i] == '/' { i++ } - name, z.p0 = z.lower(i) + name, z.p0 = z.word(i, true) hasAttr = z.p0 != z.p1 return } @@ -397,23 +426,36 @@ func (z *Tokenizer) TagName() (name []byte, hasAttr bool) { // attribute for the current tag token and whether there are more attributes. // The contents of the returned slices may change on the next call to Next. func (z *Tokenizer) TagAttr() (key, val []byte, moreAttr bool) { - key, i := z.lower(z.p0) - // Get past the "=\"". - if i == z.p1 || z.buf[i] != '=' { + key, i := z.attrName(z.p0) + // Check for an empty attribute value. + if i == z.p1 { + z.p0 = i + return + } + // Get past the equals and quote characters. + if z.buf[i] != '=' { + z.p0, moreAttr = i, true return } i = z.trim(i + 1) - if i == z.p1 || z.buf[i] != '"' { + if i == z.p1 { + z.p0 = i + return + } + closeQuote := z.buf[i] + if closeQuote != '\'' && closeQuote != '"' { + val, z.p0 = z.word(i, false) + moreAttr = z.p0 != z.p1 return } i = z.trim(i + 1) - // Copy and unescape everything up to the closing '"'. + // Copy and unescape everything up to the closing quote. dst, src := i, i loop: for src < z.p1 { c := z.buf[src] switch c { - case '"': + case closeQuote: src++ break loop case '&': diff --git a/src/pkg/html/token_test.go b/src/pkg/html/token_test.go index 5cf1f6dac..c17b436aa 100644 --- a/src/pkg/html/token_test.go +++ b/src/pkg/html/token_test.go @@ -100,13 +100,61 @@ var tokenTests = []tokenTest{ "<p \t\n iD=\"a"B\" foo=\"bar\"><EM>te<&;xt</em></p>", `<p id="a"B" foo="bar">$<em>$te<&;xt$</em>$</p>`, }, - // A non-existant entity. Tokenizing and converting back to a string should + // A nonexistent entity. Tokenizing and converting back to a string should // escape the "&" to become "&". { "noSuchEntity", `<a b="c&noSuchEntity;d"><&alsoDoesntExist;&`, `<a b="c&noSuchEntity;d">$<&alsoDoesntExist;&`, }, + + // Attribute tests: + // http://dev.w3.org/html5/spec/Overview.html#attributes-0 + { + "Empty attribute", + `<input disabled FOO>`, + `<input disabled="" foo="">`, + }, + { + "Empty attribute, whitespace", + `<input disabled FOO >`, + `<input disabled="" foo="">`, + }, + { + "Unquoted attribute value", + `<input value=yes FOO=BAR>`, + `<input value="yes" foo="BAR">`, + }, + { + "Unquoted attribute value, spaces", + `<input value = yes FOO = BAR>`, + `<input value="yes" foo="BAR">`, + }, + { + "Unquoted attribute value, trailing space", + `<input value=yes FOO=BAR >`, + `<input value="yes" foo="BAR">`, + }, + { + "Single-quoted attribute value", + `<input value='yes' FOO='BAR'>`, + `<input value="yes" foo="BAR">`, + }, + { + "Single-quoted attribute value, trailing space", + `<input value='yes' FOO='BAR' >`, + `<input value="yes" foo="BAR">`, + }, + { + "Double-quoted attribute value", + `<input value="I'm an attribute" FOO="BAR">`, + `<input value="I'm an attribute" foo="BAR">`, + }, + { + "Attribute name characters", + `<meta http-equiv="content-type">`, + `<meta http-equiv="content-type">`, + }, } func TestTokenizer(t *testing.T) { diff --git a/src/pkg/http/cgi/child.go b/src/pkg/http/cgi/child.go index c7d48b9eb..e1ad7ad32 100644 --- a/src/pkg/http/cgi/child.go +++ b/src/pkg/http/cgi/child.go @@ -9,10 +9,12 @@ package cgi import ( "bufio" + "crypto/tls" "fmt" "http" "io" "io/ioutil" + "net" "os" "strconv" "strings" @@ -21,8 +23,16 @@ import ( // Request returns the HTTP request as represented in the current // environment. This assumes the current program is being run // by a web server in a CGI environment. +// The returned Request's Body is populated, if applicable. func Request() (*http.Request, os.Error) { - return requestFromEnvironment(envMap(os.Environ())) + r, err := RequestFromMap(envMap(os.Environ())) + if err != nil { + return nil, err + } + if r.ContentLength > 0 { + r.Body = ioutil.NopCloser(io.LimitReader(os.Stdin, r.ContentLength)) + } + return r, nil } func envMap(env []string) map[string]string { @@ -42,37 +52,44 @@ var skipHeader = map[string]bool{ "HTTP_USER_AGENT": true, } -func requestFromEnvironment(env map[string]string) (*http.Request, os.Error) { +// RequestFromMap creates an http.Request from CGI variables. +// The returned Request's Body field is not populated. +func RequestFromMap(params map[string]string) (*http.Request, os.Error) { r := new(http.Request) - r.Method = env["REQUEST_METHOD"] + r.Method = params["REQUEST_METHOD"] if r.Method == "" { return nil, os.NewError("cgi: no REQUEST_METHOD in environment") } + + r.Proto = params["SERVER_PROTOCOL"] + var ok bool + r.ProtoMajor, r.ProtoMinor, ok = http.ParseHTTPVersion(r.Proto) + if !ok { + return nil, os.NewError("cgi: invalid SERVER_PROTOCOL version") + } + r.Close = true r.Trailer = http.Header{} r.Header = http.Header{} - r.Host = env["HTTP_HOST"] - r.Referer = env["HTTP_REFERER"] - r.UserAgent = env["HTTP_USER_AGENT"] + r.Host = params["HTTP_HOST"] + r.Referer = params["HTTP_REFERER"] + r.UserAgent = params["HTTP_USER_AGENT"] - // CGI doesn't allow chunked requests, so these should all be accurate: - r.Proto = "HTTP/1.0" - r.ProtoMajor = 1 - r.ProtoMinor = 0 - r.TransferEncoding = nil - - if lenstr := env["CONTENT_LENGTH"]; lenstr != "" { + if lenstr := params["CONTENT_LENGTH"]; lenstr != "" { clen, err := strconv.Atoi64(lenstr) if err != nil { return nil, os.NewError("cgi: bad CONTENT_LENGTH in environment: " + lenstr) } r.ContentLength = clen - r.Body = ioutil.NopCloser(io.LimitReader(os.Stdin, clen)) + } + + if ct := params["CONTENT_TYPE"]; ct != "" { + r.Header.Set("Content-Type", ct) } // Copy "HTTP_FOO_BAR" variables to "Foo-Bar" Headers - for k, v := range env { + for k, v := range params { if !strings.HasPrefix(k, "HTTP_") || skipHeader[k] { continue } @@ -84,7 +101,7 @@ func requestFromEnvironment(env map[string]string) (*http.Request, os.Error) { if r.Host != "" { // Hostname is provided, so we can reasonably construct a URL, // even if we have to assume 'http' for the scheme. - r.RawURL = "http://" + r.Host + env["REQUEST_URI"] + r.RawURL = "http://" + r.Host + params["REQUEST_URI"] url, err := http.ParseURL(r.RawURL) if err != nil { return nil, os.NewError("cgi: failed to parse host and REQUEST_URI into a URL: " + r.RawURL) @@ -94,13 +111,25 @@ func requestFromEnvironment(env map[string]string) (*http.Request, os.Error) { // Fallback logic if we don't have a Host header or the URL // failed to parse if r.URL == nil { - r.RawURL = env["REQUEST_URI"] + r.RawURL = params["REQUEST_URI"] url, err := http.ParseURL(r.RawURL) if err != nil { return nil, os.NewError("cgi: failed to parse REQUEST_URI into a URL: " + r.RawURL) } r.URL = url } + + // There's apparently a de-facto standard for this. + // http://docstore.mik.ua/orelly/linux/cgi/ch03_02.htm#ch03-35636 + if s := params["HTTPS"]; s == "on" || s == "ON" || s == "1" { + r.TLS = &tls.ConnectionState{HandshakeComplete: true} + } + + // Request.RemoteAddr has its port set by Go's standard http + // server, so we do here too. We don't have one, though, so we + // use a dummy one. + r.RemoteAddr = net.JoinHostPort(params["REMOTE_ADDR"], "0") + return r, nil } @@ -139,10 +168,6 @@ func (r *response) Flush() { r.bufw.Flush() } -func (r *response) RemoteAddr() string { - return os.Getenv("REMOTE_ADDR") -} - func (r *response) Header() http.Header { return r.header } @@ -168,25 +193,7 @@ func (r *response) WriteHeader(code int) { r.header.Add("Content-Type", "text/html; charset=utf-8") } - // TODO: add a method on http.Header to write itself to an io.Writer? - // This is duplicated code. - for k, vv := range r.header { - for _, v := range vv { - v = strings.Replace(v, "\n", "", -1) - v = strings.Replace(v, "\r", "", -1) - v = strings.TrimSpace(v) - fmt.Fprintf(r.bufw, "%s: %s\r\n", k, v) - } - } - r.bufw.Write([]byte("\r\n")) + r.header.Write(r.bufw) + r.bufw.WriteString("\r\n") r.bufw.Flush() } - -func (r *response) UsingTLS() bool { - // There's apparently a de-facto standard for this. - // http://docstore.mik.ua/orelly/linux/cgi/ch03_02.htm#ch03-35636 - if s := os.Getenv("HTTPS"); s == "on" || s == "ON" || s == "1" { - return true - } - return false -} diff --git a/src/pkg/http/cgi/child_test.go b/src/pkg/http/cgi/child_test.go index db0e09cf6..d12947814 100644 --- a/src/pkg/http/cgi/child_test.go +++ b/src/pkg/http/cgi/child_test.go @@ -12,6 +12,7 @@ import ( func TestRequest(t *testing.T) { env := map[string]string{ + "SERVER_PROTOCOL": "HTTP/1.1", "REQUEST_METHOD": "GET", "HTTP_HOST": "example.com", "HTTP_REFERER": "elsewhere", @@ -19,10 +20,13 @@ func TestRequest(t *testing.T) { "HTTP_FOO_BAR": "baz", "REQUEST_URI": "/path?a=b", "CONTENT_LENGTH": "123", + "CONTENT_TYPE": "text/xml", + "HTTPS": "1", + "REMOTE_ADDR": "5.6.7.8", } - req, err := requestFromEnvironment(env) + req, err := RequestFromMap(env) if err != nil { - t.Fatalf("requestFromEnvironment: %v", err) + t.Fatalf("RequestFromMap: %v", err) } if g, e := req.UserAgent, "goclient"; e != g { t.Errorf("expected UserAgent %q; got %q", e, g) @@ -34,6 +38,9 @@ func TestRequest(t *testing.T) { // Tests that we don't put recognized headers in the map t.Errorf("expected User-Agent %q; got %q", e, g) } + if g, e := req.Header.Get("Content-Type"), "text/xml"; e != g { + t.Errorf("expected Content-Type %q; got %q", e, g) + } if g, e := req.ContentLength, int64(123); e != g { t.Errorf("expected ContentLength %d; got %d", e, g) } @@ -58,18 +65,25 @@ func TestRequest(t *testing.T) { if req.Trailer == nil { t.Errorf("unexpected nil Trailer") } + if req.TLS == nil { + t.Errorf("expected non-nil TLS") + } + if e, g := "5.6.7.8:0", req.RemoteAddr; e != g { + t.Errorf("RemoteAddr: got %q; want %q", g, e) + } } func TestRequestWithoutHost(t *testing.T) { env := map[string]string{ - "HTTP_HOST": "", - "REQUEST_METHOD": "GET", - "REQUEST_URI": "/path?a=b", - "CONTENT_LENGTH": "123", + "SERVER_PROTOCOL": "HTTP/1.1", + "HTTP_HOST": "", + "REQUEST_METHOD": "GET", + "REQUEST_URI": "/path?a=b", + "CONTENT_LENGTH": "123", } - req, err := requestFromEnvironment(env) + req, err := RequestFromMap(env) if err != nil { - t.Fatalf("requestFromEnvironment: %v", err) + t.Fatalf("RequestFromMap: %v", err) } if g, e := req.RawURL, "/path?a=b"; e != g { t.Errorf("expected RawURL %q; got %q", e, g) diff --git a/src/pkg/http/cgi/host.go b/src/pkg/http/cgi/host.go index 136d4e4ee..7ab3f9247 100644 --- a/src/pkg/http/cgi/host.go +++ b/src/pkg/http/cgi/host.go @@ -36,7 +36,9 @@ var osDefaultInheritEnv = map[string][]string{ "darwin": []string{"DYLD_LIBRARY_PATH"}, "freebsd": []string{"LD_LIBRARY_PATH"}, "hpux": []string{"LD_LIBRARY_PATH", "SHLIB_PATH"}, + "irix": []string{"LD_LIBRARY_PATH", "LD_LIBRARYN32_PATH", "LD_LIBRARY64_PATH"}, "linux": []string{"LD_LIBRARY_PATH"}, + "solaris": []string{"LD_LIBRARY_PATH", "LD_LIBRARY_PATH_32", "LD_LIBRARY_PATH_64"}, "windows": []string{"SystemRoot", "COMSPEC", "PATHEXT", "WINDIR"}, } @@ -86,6 +88,7 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { env := []string{ "SERVER_SOFTWARE=go", "SERVER_NAME=" + req.Host, + "SERVER_PROTOCOL=HTTP/1.1", "HTTP_HOST=" + req.Host, "GATEWAY_INTERFACE=CGI/1.1", "REQUEST_METHOD=" + req.Method, @@ -153,34 +156,35 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { cwd = "." } - args := []string{h.Path} - args = append(args, h.Args...) - - cmd, err := exec.Run( - pathBase, - args, - env, - cwd, - exec.Pipe, // stdin - exec.Pipe, // stdout - exec.PassThrough, // stderr (for now) - ) - if err != nil { + internalError := func(err os.Error) { rw.WriteHeader(http.StatusInternalServerError) h.printf("CGI error: %v", err) - return } - defer func() { - cmd.Stdin.Close() - cmd.Stdout.Close() - cmd.Wait(0) // no zombies - }() + cmd := &exec.Cmd{ + Path: pathBase, + Args: append([]string{h.Path}, h.Args...), + Dir: cwd, + Env: env, + Stderr: os.Stderr, // for now + } if req.ContentLength != 0 { - go io.Copy(cmd.Stdin, req.Body) + cmd.Stdin = req.Body + } + stdoutRead, err := cmd.StdoutPipe() + if err != nil { + internalError(err) + return + } + + err = cmd.Start() + if err != nil { + internalError(err) + return } + defer cmd.Wait() - linebody, _ := bufio.NewReaderSize(cmd.Stdout, 1024) + linebody, _ := bufio.NewReaderSize(stdoutRead, 1024) headers := make(http.Header) statusCode := 0 for { diff --git a/src/pkg/http/cgi/host_test.go b/src/pkg/http/cgi/host_test.go index 9ac085f2f..bbdb715cf 100644 --- a/src/pkg/http/cgi/host_test.go +++ b/src/pkg/http/cgi/host_test.go @@ -17,20 +17,6 @@ import ( "testing" ) -var cgiScriptWorks = canRun("./testdata/test.cgi") - -func canRun(s string) bool { - c, err := exec.Run(s, []string{s}, nil, ".", exec.DevNull, exec.DevNull, exec.DevNull) - if err != nil { - return false - } - w, err := c.Wait(0) - if err != nil { - return false - } - return w.Exited() && w.ExitStatus() == 0 -} - func newRequest(httpreq string) *http.Request { buf := bufio.NewReader(strings.NewReader(httpreq)) req, err := http.ReadRequest(buf) @@ -76,8 +62,15 @@ readlines: return rw } +var cgiTested = false +var cgiWorks bool + func skipTest(t *testing.T) bool { - if !cgiScriptWorks { + if !cgiTested { + cgiTested = true + cgiWorks = exec.Command("./testdata/test.cgi").Run() == nil + } + if !cgiWorks { // No Perl on Windows, needed by test.cgi // TODO: make the child process be Go, not Perl. t.Logf("Skipping test: test.cgi failed.") diff --git a/src/pkg/http/chunked.go b/src/pkg/http/chunked.go index 66195f06b..59121c5a2 100644 --- a/src/pkg/http/chunked.go +++ b/src/pkg/http/chunked.go @@ -6,19 +6,29 @@ package http import ( "io" + "log" "os" "strconv" ) // NewChunkedWriter returns a new writer that translates writes into HTTP -// "chunked" format before writing them to w. Closing the returned writer +// "chunked" format before writing them to w. Closing the returned writer // sends the final 0-length chunk that marks the end of the stream. +// +// NewChunkedWriter is not needed by normal applications. The http +// package adds chunking automatically if handlers don't set a +// Content-Length header. Using NewChunkedWriter inside a handler +// would result in double chunking or chunking with a Content-Length +// length, both of which are wrong. func NewChunkedWriter(w io.Writer) io.WriteCloser { + if _, bad := w.(*response); bad { + log.Printf("warning: using NewChunkedWriter in an http.Handler; expect corrupt output") + } return &chunkedWriter{w} } // Writing to ChunkedWriter translates to writing in HTTP chunked Transfer -// Encoding wire format to the undering Wire writer. +// Encoding wire format to the underlying Wire writer. type chunkedWriter struct { Wire io.Writer } diff --git a/src/pkg/http/client.go b/src/pkg/http/client.go index d73cbc855..71b037042 100644 --- a/src/pkg/http/client.go +++ b/src/pkg/http/client.go @@ -7,13 +7,10 @@ package http import ( - "bytes" "encoding/base64" "fmt" "io" - "io/ioutil" "os" - "strconv" "strings" ) @@ -74,6 +71,9 @@ type readClose struct { // // Generally Get, Post, or PostForm will be used instead of Do. func (c *Client) Do(req *Request) (resp *Response, err os.Error) { + if req.Method == "GET" || req.Method == "HEAD" { + return c.doFollowingRedirects(req) + } return send(req, c.Transport) } @@ -97,13 +97,10 @@ func send(req *Request, t RoundTripper) (resp *Response, err os.Error) { info := req.URL.RawUserinfo if len(info) > 0 { - enc := base64.URLEncoding - encoded := make([]byte, enc.EncodedLen(len(info))) - enc.Encode(encoded, []byte(info)) if req.Header == nil { req.Header = make(Header) } - req.Header.Set("Authorization", "Basic "+string(encoded)) + req.Header.Set("Authorization", "Basic "+base64.URLEncoding.EncodeToString([]byte(info))) } return t.RoundTrip(req) } @@ -126,13 +123,10 @@ func shouldRedirect(statusCode int) bool { // 303 (See Other) // 307 (Temporary Redirect) // -// finalURL is the URL from which the response was fetched -- identical to the -// input URL unless redirects were followed. -// // Caller should close r.Body when done reading from it. // // Get is a convenience wrapper around DefaultClient.Get. -func Get(url string) (r *Response, finalURL string, err os.Error) { +func Get(url string) (r *Response, err os.Error) { return DefaultClient.Get(url) } @@ -145,11 +139,16 @@ func Get(url string) (r *Response, finalURL string, err os.Error) { // 303 (See Other) // 307 (Temporary Redirect) // -// finalURL is the URL from which the response was fetched -- identical -// to the input URL unless redirects were followed. -// // Caller should close r.Body when done reading from it. -func (c *Client) Get(url string) (r *Response, finalURL string, err os.Error) { +func (c *Client) Get(url string) (r *Response, err os.Error) { + req, err := NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + return c.doFollowingRedirects(req) +} + +func (c *Client) doFollowingRedirects(ireq *Request) (r *Response, err os.Error) { // TODO: if/when we add cookie support, the redirected request shouldn't // necessarily supply the same cookies as the original. var base *URL @@ -159,33 +158,33 @@ func (c *Client) Get(url string) (r *Response, finalURL string, err os.Error) { } var via []*Request + req := ireq + url := "" // next relative or absolute URL to fetch (after first request) for redirect := 0; ; redirect++ { - var req Request - req.Method = "GET" - req.Header = make(Header) - if base == nil { - req.URL, err = ParseURL(url) - } else { + if redirect != 0 { + req = new(Request) + req.Method = ireq.Method + req.Header = make(Header) req.URL, err = base.ParseURL(url) - } - if err != nil { - break - } - if len(via) > 0 { - // Add the Referer header. - lastReq := via[len(via)-1] - if lastReq.URL.Scheme != "https" { - req.Referer = lastReq.URL.String() - } - - err = redirectChecker(&req, via) if err != nil { break } + if len(via) > 0 { + // Add the Referer header. + lastReq := via[len(via)-1] + if lastReq.URL.Scheme != "https" { + req.Referer = lastReq.URL.String() + } + + err = redirectChecker(req, via) + if err != nil { + break + } + } } url = req.URL.String() - if r, err = send(&req, c.Transport); err != nil { + if r, err = send(req, c.Transport); err != nil { break } if shouldRedirect(r.StatusCode) { @@ -195,14 +194,14 @@ func (c *Client) Get(url string) (r *Response, finalURL string, err os.Error) { break } base = req.URL - via = append(via, &req) + via = append(via, req) continue } - finalURL = url return } - err = &URLError{"Get", url, err} + method := ireq.Method + err = &URLError{method[0:1] + strings.ToLower(method[1:]), url, err} return } @@ -226,23 +225,12 @@ func Post(url string, bodyType string, body io.Reader) (r *Response, err os.Erro // // Caller should close r.Body when done reading from it. func (c *Client) Post(url string, bodyType string, body io.Reader) (r *Response, err os.Error) { - var req Request - req.Method = "POST" - req.ProtoMajor = 1 - req.ProtoMinor = 1 - req.Close = true - req.Body = ioutil.NopCloser(body) - req.Header = Header{ - "Content-Type": {bodyType}, - } - req.TransferEncoding = []string{"chunked"} - - req.URL, err = ParseURL(url) + req, err := NewRequest("POST", url, body) if err != nil { return nil, err } - - return send(&req, c.Transport) + req.Header.Set("Content-Type", bodyType) + return send(req, c.Transport) } // PostForm issues a POST to the specified URL, @@ -251,7 +239,7 @@ func (c *Client) Post(url string, bodyType string, body io.Reader) (r *Response, // Caller should close r.Body when done reading from it. // // PostForm is a wrapper around DefaultClient.PostForm -func PostForm(url string, data map[string]string) (r *Response, err os.Error) { +func PostForm(url string, data Values) (r *Response, err os.Error) { return DefaultClient.PostForm(url, data) } @@ -259,50 +247,36 @@ func PostForm(url string, data map[string]string) (r *Response, err os.Error) { // with data's keys and values urlencoded as the request body. // // Caller should close r.Body when done reading from it. -func (c *Client) PostForm(url string, data map[string]string) (r *Response, err os.Error) { - var req Request - req.Method = "POST" - req.ProtoMajor = 1 - req.ProtoMinor = 1 - req.Close = true - body := urlencode(data) - req.Body = ioutil.NopCloser(body) - req.Header = Header{ - "Content-Type": {"application/x-www-form-urlencoded"}, - "Content-Length": {strconv.Itoa(body.Len())}, - } - req.ContentLength = int64(body.Len()) - - req.URL, err = ParseURL(url) - if err != nil { - return nil, err - } - - return send(&req, c.Transport) +func (c *Client) PostForm(url string, data Values) (r *Response, err os.Error) { + return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) } -// TODO: remove this function when PostForm takes a multimap. -func urlencode(data map[string]string) (b *bytes.Buffer) { - m := make(map[string][]string, len(data)) - for k, v := range data { - m[k] = []string{v} - } - return bytes.NewBuffer([]byte(EncodeQuery(m))) -} - -// Head issues a HEAD to the specified URL. +// Head issues a HEAD to the specified URL. If the response is one of the +// following redirect codes, Head follows the redirect after calling the +// Client's CheckRedirect function. +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) // // Head is a wrapper around DefaultClient.Head func Head(url string) (r *Response, err os.Error) { return DefaultClient.Head(url) } -// Head issues a HEAD to the specified URL. +// Head issues a HEAD to the specified URL. If the response is one of the +// following redirect codes, Head follows the redirect after calling the +// Client's CheckRedirect function. +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) func (c *Client) Head(url string) (r *Response, err os.Error) { - var req Request - req.Method = "HEAD" - if req.URL, err = ParseURL(url); err != nil { - return + req, err := NewRequest("HEAD", url, nil) + if err != nil { + return nil, err } - return send(&req, c.Transport) + return c.doFollowingRedirects(req) } diff --git a/src/pkg/http/client_test.go b/src/pkg/http/client_test.go index 59d62c1c9..9ef81d9d4 100644 --- a/src/pkg/http/client_test.go +++ b/src/pkg/http/client_test.go @@ -10,6 +10,7 @@ import ( "fmt" . "http" "http/httptest" + "io" "io/ioutil" "os" "strconv" @@ -26,7 +27,7 @@ func TestClient(t *testing.T) { ts := httptest.NewServer(robotsTxtHandler) defer ts.Close() - r, _, err := Get(ts.URL) + r, err := Get(ts.URL) var b []byte if err == nil { b, err = ioutil.ReadAll(r.Body) @@ -77,6 +78,71 @@ func TestGetRequestFormat(t *testing.T) { } } +func TestPostRequestFormat(t *testing.T) { + tr := &recordingTransport{} + client := &Client{Transport: tr} + + url := "http://dummy.faketld/" + json := `{"key":"value"}` + b := strings.NewReader(json) + client.Post(url, "application/json", b) // Note: doesn't hit network + + if tr.req.Method != "POST" { + t.Errorf("got method %q, want %q", tr.req.Method, "POST") + } + if tr.req.URL.String() != url { + t.Errorf("got URL %q, want %q", tr.req.URL.String(), url) + } + if tr.req.Header == nil { + t.Fatalf("expected non-nil request Header") + } + if tr.req.Close { + t.Error("got Close true, want false") + } + if g, e := tr.req.ContentLength, int64(len(json)); g != e { + t.Errorf("got ContentLength %d, want %d", g, e) + } +} + +func TestPostFormRequestFormat(t *testing.T) { + tr := &recordingTransport{} + client := &Client{Transport: tr} + + url := "http://dummy.faketld/" + form := make(Values) + form.Set("foo", "bar") + form.Add("foo", "bar2") + form.Set("bar", "baz") + client.PostForm(url, form) // Note: doesn't hit network + + if tr.req.Method != "POST" { + t.Errorf("got method %q, want %q", tr.req.Method, "POST") + } + if tr.req.URL.String() != url { + t.Errorf("got URL %q, want %q", tr.req.URL.String(), url) + } + if tr.req.Header == nil { + t.Fatalf("expected non-nil request Header") + } + if g, e := tr.req.Header.Get("Content-Type"), "application/x-www-form-urlencoded"; g != e { + t.Errorf("got Content-Type %q, want %q", g, e) + } + if tr.req.Close { + t.Error("got Close true, want false") + } + expectedBody := "foo=bar&foo=bar2&bar=baz" + if g, e := tr.req.ContentLength, int64(len(expectedBody)); g != e { + t.Errorf("got ContentLength %d, want %d", g, e) + } + bodyb, err := ioutil.ReadAll(tr.req.Body) + if err != nil { + t.Fatalf("ReadAll on req.Body: %v", err) + } + if g := string(bodyb); g != expectedBody { + t.Errorf("got body %q, want %q", g, expectedBody) + } +} + func TestRedirects(t *testing.T) { var ts *httptest.Server ts = httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { @@ -96,9 +162,22 @@ func TestRedirects(t *testing.T) { defer ts.Close() c := &Client{} - _, _, err := c.Get(ts.URL) + _, err := c.Get(ts.URL) if e, g := "Get /?n=10: stopped after 10 redirects", fmt.Sprintf("%v", err); e != g { - t.Errorf("with default client, expected error %q, got %q", e, g) + t.Errorf("with default client Get, expected error %q, got %q", e, g) + } + + // HEAD request should also have the ability to follow redirects. + _, err = c.Head(ts.URL) + if e, g := "Head /?n=10: stopped after 10 redirects", fmt.Sprintf("%v", err); e != g { + t.Errorf("with default client Head, expected error %q, got %q", e, g) + } + + // Do should also follow redirects. + greq, _ := NewRequest("GET", ts.URL, nil) + _, err = c.Do(greq) + if e, g := "Get /?n=10: stopped after 10 redirects", fmt.Sprintf("%v", err); e != g { + t.Errorf("with default client Do, expected error %q, got %q", e, g) } var checkErr os.Error @@ -107,7 +186,8 @@ func TestRedirects(t *testing.T) { lastVia = via return checkErr }} - _, finalUrl, err := c.Get(ts.URL) + res, err := c.Get(ts.URL) + finalUrl := res.Request.URL.String() if e, g := "<nil>", fmt.Sprintf("%v", err); e != g { t.Errorf("with custom client, expected error %q, got %q", e, g) } @@ -119,8 +199,47 @@ func TestRedirects(t *testing.T) { } checkErr = os.NewError("no redirects allowed") - _, finalUrl, err = c.Get(ts.URL) + res, err = c.Get(ts.URL) + finalUrl = res.Request.URL.String() if e, g := "Get /?n=1: no redirects allowed", fmt.Sprintf("%v", err); e != g { t.Errorf("with redirects forbidden, expected error %q, got %q", e, g) } } + +func TestStreamingGet(t *testing.T) { + say := make(chan string) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.(Flusher).Flush() + for str := range say { + w.Write([]byte(str)) + w.(Flusher).Flush() + } + })) + defer ts.Close() + + c := &Client{} + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + var buf [10]byte + for _, str := range []string{"i", "am", "also", "known", "as", "comet"} { + say <- str + n, err := io.ReadFull(res.Body, buf[0:len(str)]) + if err != nil { + t.Fatalf("ReadFull on %q: %v", str, err) + } + if n != len(str) { + t.Fatalf("Receiving %q, only read %d bytes", str, n) + } + got := string(buf[0:n]) + if got != str { + t.Fatalf("Expected %q, got %q", str, got) + } + } + close(say) + _, err = io.ReadFull(res.Body, buf[0:1]) + if err != os.EOF { + t.Fatalf("at end expected EOF, got %v", err) + } +} diff --git a/src/pkg/http/cookie.go b/src/pkg/http/cookie.go index 2c01826a1..eb61a7001 100644 --- a/src/pkg/http/cookie.go +++ b/src/pkg/http/cookie.go @@ -15,9 +15,9 @@ import ( "time" ) -// This implementation is done according to IETF draft-ietf-httpstate-cookie-23, found at +// This implementation is done according to RFC 6265: // -// http://tools.ietf.org/html/draft-ietf-httpstate-cookie-23 +// http://tools.ietf.org/html/rfc6265 // A Cookie represents an HTTP cookie as sent in the Set-Cookie header of an // HTTP response or the Cookie header of an HTTP request. @@ -81,12 +81,17 @@ func readSetCookies(h Header) []*Cookie { if j := strings.Index(attr, "="); j >= 0 { attr, val = attr[:j], attr[j+1:] } - val, success = parseCookieValue(val) + lowerAttr := strings.ToLower(attr) + parseCookieValueFn := parseCookieValue + if lowerAttr == "expires" { + parseCookieValueFn = parseCookieExpiresValue + } + val, success = parseCookieValueFn(val) if !success { c.Unparsed = append(c.Unparsed, parts[i]) continue } - switch strings.ToLower(attr) { + switch lowerAttr { case "secure": c.Secure = true continue @@ -112,8 +117,11 @@ func readSetCookies(h Header) []*Cookie { c.RawExpires = val exptime, err := time.Parse(time.RFC1123, val) if err != nil { - c.Expires = time.Time{} - break + exptime, err = time.Parse("Mon, 02-Jan-2006 15:04:05 MST", val) + if err != nil { + c.Expires = time.Time{} + break + } } c.Expires = *exptime continue @@ -130,6 +138,37 @@ func readSetCookies(h Header) []*Cookie { return cookies } +// SetCookie adds a Set-Cookie header to the provided ResponseWriter's headers. +func SetCookie(w ResponseWriter, cookie *Cookie) { + var b bytes.Buffer + writeSetCookieToBuffer(&b, cookie) + w.Header().Add("Set-Cookie", b.String()) +} + +func writeSetCookieToBuffer(buf *bytes.Buffer, c *Cookie) { + fmt.Fprintf(buf, "%s=%s", sanitizeName(c.Name), sanitizeValue(c.Value)) + if len(c.Path) > 0 { + fmt.Fprintf(buf, "; Path=%s", sanitizeValue(c.Path)) + } + if len(c.Domain) > 0 { + fmt.Fprintf(buf, "; Domain=%s", sanitizeValue(c.Domain)) + } + if len(c.Expires.Zone) > 0 { + fmt.Fprintf(buf, "; Expires=%s", c.Expires.Format(time.RFC1123)) + } + if c.MaxAge > 0 { + fmt.Fprintf(buf, "; Max-Age=%d", c.MaxAge) + } else if c.MaxAge < 0 { + fmt.Fprintf(buf, "; Max-Age=0") + } + if c.HttpOnly { + fmt.Fprintf(buf, "; HttpOnly") + } + if c.Secure { + fmt.Fprintf(buf, "; Secure") + } +} + // writeSetCookies writes the wire representation of the set-cookies // to w. Each cookie is written on a separate "Set-Cookie: " line. // This choice is made because HTTP parsers tend to have a limit on @@ -142,27 +181,7 @@ func writeSetCookies(w io.Writer, kk []*Cookie) os.Error { var b bytes.Buffer for _, c := range kk { b.Reset() - fmt.Fprintf(&b, "%s=%s", sanitizeName(c.Name), sanitizeValue(c.Value)) - if len(c.Path) > 0 { - fmt.Fprintf(&b, "; Path=%s", sanitizeValue(c.Path)) - } - if len(c.Domain) > 0 { - fmt.Fprintf(&b, "; Domain=%s", sanitizeValue(c.Domain)) - } - if len(c.Expires.Zone) > 0 { - fmt.Fprintf(&b, "; Expires=%s", c.Expires.Format(time.RFC1123)) - } - if c.MaxAge > 0 { - fmt.Fprintf(&b, "; Max-Age=%d", c.MaxAge) - } else if c.MaxAge < 0 { - fmt.Fprintf(&b, "; Max-Age=0") - } - if c.HttpOnly { - fmt.Fprintf(&b, "; HttpOnly") - } - if c.Secure { - fmt.Fprintf(&b, "; Secure") - } + writeSetCookieToBuffer(&b, c) lines = append(lines, "Set-Cookie: "+b.String()+"\r\n") } sort.SortStrings(lines) @@ -218,22 +237,26 @@ func readCookies(h Header) []*Cookie { return cookies } -// writeCookies writes the wire representation of the cookies -// to w. Each cookie is written on a separate "Cookie: " line. -// This choice is made because HTTP parsers tend to have a limit on -// line-length, so it seems safer to place cookies on separate lines. +// writeCookies writes the wire representation of the cookies to +// w. According to RFC 6265 section 5.4, writeCookies does not +// attach more than one Cookie header field. That means all +// cookies, if any, are written into the same line, separated by +// semicolon. func writeCookies(w io.Writer, kk []*Cookie) os.Error { - lines := make([]string, 0, len(kk)) - for _, c := range kk { - lines = append(lines, fmt.Sprintf("Cookie: %s=%s\r\n", sanitizeName(c.Name), sanitizeValue(c.Value))) + if len(kk) == 0 { + return nil } - sort.SortStrings(lines) - for _, l := range lines { - if _, err := io.WriteString(w, l); err != nil { - return err + var buf bytes.Buffer + fmt.Fprintf(&buf, "Cookie: ") + for i, c := range kk { + if i > 0 { + fmt.Fprintf(&buf, "; ") } + fmt.Fprintf(&buf, "%s=%s", sanitizeName(c.Name), sanitizeValue(c.Value)) } - return nil + fmt.Fprintf(&buf, "\r\n") + _, err := w.Write(buf.Bytes()) + return err } func sanitizeName(n string) string { @@ -257,7 +280,7 @@ func unquoteCookieValue(v string) string { } func isCookieByte(c byte) bool { - switch true { + switch { case c == 0x21, 0x23 <= c && c <= 0x2b, 0x2d <= c && c <= 0x3a, 0x3c <= c && c <= 0x5b, 0x5d <= c && c <= 0x7e: return true @@ -265,10 +288,22 @@ func isCookieByte(c byte) bool { return false } +func isCookieExpiresByte(c byte) (ok bool) { + return isCookieByte(c) || c == ',' || c == ' ' +} + func parseCookieValue(raw string) (string, bool) { + return parseCookieValueUsing(raw, isCookieByte) +} + +func parseCookieExpiresValue(raw string) (string, bool) { + return parseCookieValueUsing(raw, isCookieExpiresByte) +} + +func parseCookieValueUsing(raw string, validByte func(byte) bool) (string, bool) { raw = unquoteCookieValue(raw) for i := 0; i < len(raw); i++ { - if !isCookieByte(raw[i]) { + if !validByte(raw[i]) { return "", false } } diff --git a/src/pkg/http/cookie_test.go b/src/pkg/http/cookie_test.go index a3ae85cd6..02e42226b 100644 --- a/src/pkg/http/cookie_test.go +++ b/src/pkg/http/cookie_test.go @@ -8,11 +8,12 @@ import ( "bytes" "fmt" "json" + "os" "reflect" "testing" + "time" ) - var writeSetCookiesTests = []struct { Cookies []*Cookie Raw string @@ -43,14 +44,55 @@ func TestWriteSetCookies(t *testing.T) { } } +type headerOnlyResponseWriter Header + +func (ho headerOnlyResponseWriter) Header() Header { + return Header(ho) +} + +func (ho headerOnlyResponseWriter) Write([]byte) (int, os.Error) { + panic("NOIMPL") +} + +func (ho headerOnlyResponseWriter) WriteHeader(int) { + panic("NOIMPL") +} + +func TestSetCookie(t *testing.T) { + m := make(Header) + SetCookie(headerOnlyResponseWriter(m), &Cookie{Name: "cookie-1", Value: "one", Path: "/restricted/"}) + SetCookie(headerOnlyResponseWriter(m), &Cookie{Name: "cookie-2", Value: "two", MaxAge: 3600}) + if l := len(m["Set-Cookie"]); l != 2 { + t.Fatalf("expected %d cookies, got %d", 2, l) + } + if g, e := m["Set-Cookie"][0], "cookie-1=one; Path=/restricted/"; g != e { + t.Errorf("cookie #1: want %q, got %q", e, g) + } + if g, e := m["Set-Cookie"][1], "cookie-2=two; Max-Age=3600"; g != e { + t.Errorf("cookie #2: want %q, got %q", e, g) + } +} + var writeCookiesTests = []struct { Cookies []*Cookie Raw string }{ { + []*Cookie{}, + "", + }, + { []*Cookie{&Cookie{Name: "cookie-1", Value: "v$1"}}, "Cookie: cookie-1=v$1\r\n", }, + { + []*Cookie{ + &Cookie{Name: "cookie-1", Value: "v$1"}, + &Cookie{Name: "cookie-2", Value: "v$2"}, + &Cookie{Name: "cookie-3", Value: "v$3"}, + }, + "Cookie: cookie-1=v$1; cookie-2=v$2; cookie-3=v$3\r\n", + }, } func TestWriteCookies(t *testing.T) { @@ -73,6 +115,19 @@ var readSetCookiesTests = []struct { Header{"Set-Cookie": {"Cookie-1=v$1"}}, []*Cookie{&Cookie{Name: "Cookie-1", Value: "v$1", Raw: "Cookie-1=v$1"}}, }, + { + Header{"Set-Cookie": {"NID=99=YsDT5i3E-CXax-; expires=Wed, 23-Nov-2011 01:05:03 GMT; path=/; domain=.google.ch; HttpOnly"}}, + []*Cookie{&Cookie{ + Name: "NID", + Value: "99=YsDT5i3E-CXax-", + Path: "/", + Domain: ".google.ch", + HttpOnly: true, + Expires: time.Time{Year: 2011, Month: 11, Day: 23, Hour: 1, Minute: 5, Second: 3, Weekday: 3, ZoneOffset: 0, Zone: "GMT"}, + RawExpires: "Wed, 23-Nov-2011 01:05:03 GMT", + Raw: "NID=99=YsDT5i3E-CXax-; expires=Wed, 23-Nov-2011 01:05:03 GMT; path=/; domain=.google.ch; HttpOnly", + }}, + }, } func toJSON(v interface{}) string { diff --git a/src/pkg/http/fcgi/child.go b/src/pkg/http/fcgi/child.go index 114052bee..19718824c 100644 --- a/src/pkg/http/fcgi/child.go +++ b/src/pkg/http/fcgi/child.go @@ -9,11 +9,10 @@ package fcgi import ( "fmt" "http" + "http/cgi" "io" "net" "os" - "strconv" - "strings" "time" ) @@ -38,68 +37,6 @@ func newRequest(reqId uint16, flags uint8) *request { return r } -// TODO(eds): copied from http/cgi -var skipHeader = map[string]bool{ - "HTTP_HOST": true, - "HTTP_REFERER": true, - "HTTP_USER_AGENT": true, -} - -// httpRequest converts r to an http.Request. -// TODO(eds): this is very similar to http/cgi's requestFromEnvironment -func (r *request) httpRequest(body io.ReadCloser) (*http.Request, os.Error) { - req := &http.Request{ - Method: r.params["REQUEST_METHOD"], - RawURL: r.params["REQUEST_URI"], - Body: body, - Header: http.Header{}, - Trailer: http.Header{}, - Proto: r.params["SERVER_PROTOCOL"], - } - - var ok bool - req.ProtoMajor, req.ProtoMinor, ok = http.ParseHTTPVersion(req.Proto) - if !ok { - return nil, os.NewError("fcgi: invalid HTTP version") - } - - req.Host = r.params["HTTP_HOST"] - req.Referer = r.params["HTTP_REFERER"] - req.UserAgent = r.params["HTTP_USER_AGENT"] - - if lenstr := r.params["CONTENT_LENGTH"]; lenstr != "" { - clen, err := strconv.Atoi64(r.params["CONTENT_LENGTH"]) - if err != nil { - return nil, os.NewError("fcgi: bad CONTENT_LENGTH parameter: " + lenstr) - } - req.ContentLength = clen - } - - if req.Host != "" { - req.RawURL = "http://" + req.Host + r.params["REQUEST_URI"] - url, err := http.ParseURL(req.RawURL) - if err != nil { - return nil, os.NewError("fcgi: failed to parse host and REQUEST_URI into a URL: " + req.RawURL) - } - req.URL = url - } - if req.URL == nil { - req.RawURL = r.params["REQUEST_URI"] - url, err := http.ParseURL(req.RawURL) - if err != nil { - return nil, os.NewError("fcgi: failed to parse REQUEST_URI into a URL: " + req.RawURL) - } - req.URL = url - } - - for key, val := range r.params { - if strings.HasPrefix(key, "HTTP_") && !skipHeader[key] { - req.Header.Add(strings.Replace(key[5:], "_", "-", -1), val) - } - } - return req, nil -} - // parseParams reads an encoded []byte into Params. func (r *request) parseParams() { text := r.rawParams @@ -169,15 +106,7 @@ func (r *response) WriteHeader(code int) { } fmt.Fprintf(r.w, "Status: %d %s\r\n", code, http.StatusText(code)) - // TODO(eds): this is duplicated in http and http/cgi - for k, vv := range r.header { - for _, v := range vv { - v = strings.Replace(v, "\n", "", -1) - v = strings.Replace(v, "\r", "", -1) - v = strings.TrimSpace(v) - fmt.Fprintf(r.w, "%s: %s\r\n", k, v) - } - } + r.header.Write(r.w) r.w.WriteString("\r\n") } @@ -281,12 +210,13 @@ func (c *child) serve() { func (c *child) serveRequest(req *request, body io.ReadCloser) { r := newResponse(c, req) - httpReq, err := req.httpRequest(body) + httpReq, err := cgi.RequestFromMap(req.params) if err != nil { // there was an error reading the request r.WriteHeader(http.StatusInternalServerError) c.conn.writeRecord(typeStderr, req.reqId, []byte(err.String())) } else { + httpReq.Body = body c.handler.ServeHTTP(r, httpReq) } if body != nil { diff --git a/src/pkg/http/fs.go b/src/pkg/http/fs.go index 17d5297b8..28a0c51ef 100644 --- a/src/pkg/http/fs.go +++ b/src/pkg/http/fs.go @@ -175,7 +175,9 @@ func serveFile(w ResponseWriter, r *Request, name string, redirect bool) { } w.Header().Set("Accept-Ranges", "bytes") - w.Header().Set("Content-Length", strconv.Itoa64(size)) + if w.Header().Get("Content-Encoding") == "" { + w.Header().Set("Content-Length", strconv.Itoa64(size)) + } w.WriteHeader(code) diff --git a/src/pkg/http/fs_test.go b/src/pkg/http/fs_test.go index 09d0981f2..554053449 100644 --- a/src/pkg/http/fs_test.go +++ b/src/pkg/http/fs_test.go @@ -96,12 +96,12 @@ func TestServeFileContentType(t *testing.T) { })) defer ts.Close() get := func(want string) { - resp, _, err := Get(ts.URL) + resp, err := Get(ts.URL) if err != nil { t.Fatal(err) } if h := resp.Header.Get("Content-Type"); h != want { - t.Errorf("Content-Type mismatch: got %q, want %q", h, want) + t.Errorf("Content-Type mismatch: got %d, want %d", h, want) } } get("text/plain; charset=utf-8") @@ -109,6 +109,21 @@ func TestServeFileContentType(t *testing.T) { get(ctype) } +func TestServeFileWithContentEncoding(t *testing.T) { + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Encoding", "foo") + ServeFile(w, r, "testdata/file") + })) + defer ts.Close() + resp, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + if g, e := resp.ContentLength, int64(-1); g != e { + t.Errorf("Content-Length mismatch: got %q, want %q", g, e) + } +} + func getBody(t *testing.T, req Request) (*Response, []byte) { r, err := DefaultClient.Do(&req) if err != nil { diff --git a/src/pkg/http/header.go b/src/pkg/http/header.go index 95b0f3db6..95140b01f 100644 --- a/src/pkg/http/header.go +++ b/src/pkg/http/header.go @@ -4,7 +4,14 @@ package http -import "net/textproto" +import ( + "fmt" + "io" + "net/textproto" + "os" + "sort" + "strings" +) // A Header represents the key-value pairs in an HTTP header. type Header map[string][]string @@ -35,6 +42,37 @@ func (h Header) Del(key string) { textproto.MIMEHeader(h).Del(key) } +// Write writes a header in wire format. +func (h Header) Write(w io.Writer) os.Error { + return h.WriteSubset(w, nil) +} + +// WriteSubset writes a header in wire format. +// If exclude is not nil, keys where exclude[key] == true are not written. +func (h Header) WriteSubset(w io.Writer, exclude map[string]bool) os.Error { + keys := make([]string, 0, len(h)) + for k := range h { + if exclude == nil || !exclude[k] { + keys = append(keys, k) + } + } + sort.SortStrings(keys) + for _, k := range keys { + for _, v := range h[k] { + v = strings.Replace(v, "\n", " ", -1) + v = strings.Replace(v, "\r", " ", -1) + v = strings.TrimSpace(v) + if v == "" { + continue + } + if _, err := fmt.Fprintf(w, "%s: %s\r\n", k, v); err != nil { + return err + } + } + } + return nil +} + // CanonicalHeaderKey returns the canonical format of the // header key s. The canonicalization converts the first // letter and any letter following a hyphen to upper case; diff --git a/src/pkg/http/header_test.go b/src/pkg/http/header_test.go new file mode 100644 index 000000000..7e24cb069 --- /dev/null +++ b/src/pkg/http/header_test.go @@ -0,0 +1,71 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bytes" + "testing" +) + +var headerWriteTests = []struct { + h Header + exclude map[string]bool + expected string +}{ + {Header{}, nil, ""}, + { + Header{ + "Content-Type": {"text/html; charset=UTF-8"}, + "Content-Length": {"0"}, + }, + nil, + "Content-Length: 0\r\nContent-Type: text/html; charset=UTF-8\r\n", + }, + { + Header{ + "Content-Length": {"0", "1", "2"}, + }, + nil, + "Content-Length: 0\r\nContent-Length: 1\r\nContent-Length: 2\r\n", + }, + { + Header{ + "Expires": {"-1"}, + "Content-Length": {"0"}, + "Content-Encoding": {"gzip"}, + }, + map[string]bool{"Content-Length": true}, + "Content-Encoding: gzip\r\nExpires: -1\r\n", + }, + { + Header{ + "Expires": {"-1"}, + "Content-Length": {"0", "1", "2"}, + "Content-Encoding": {"gzip"}, + }, + map[string]bool{"Content-Length": true}, + "Content-Encoding: gzip\r\nExpires: -1\r\n", + }, + { + Header{ + "Expires": {"-1"}, + "Content-Length": {"0"}, + "Content-Encoding": {"gzip"}, + }, + map[string]bool{"Content-Length": true, "Expires": true, "Content-Encoding": true}, + "", + }, +} + +func TestHeaderWrite(t *testing.T) { + var buf bytes.Buffer + for i, test := range headerWriteTests { + test.h.WriteSubset(&buf, test.exclude) + if buf.String() != test.expected { + t.Errorf("#%d:\n got: %q\nwant: %q", i, buf.String(), test.expected) + } + buf.Reset() + } +} diff --git a/src/pkg/http/httptest/server.go b/src/pkg/http/httptest/server.go index 8e385d045..879f04f33 100644 --- a/src/pkg/http/httptest/server.go +++ b/src/pkg/http/httptest/server.go @@ -108,29 +108,24 @@ func (s *Server) CloseClientConnections() { // "127.0.0.1" and "[::1]", expiring at the last second of 2049 (the end // of ASN.1 time). var localhostCert = []byte(`-----BEGIN CERTIFICATE----- -MIIBwTCCASugAwIBAgIBADALBgkqhkiG9w0BAQUwADAeFw0xMTAzMzEyMDI1MDda -Fw00OTEyMzEyMzU5NTlaMAAwggCdMAsGCSqGSIb3DQEBAQOCAIwAMIIAhwKCAIB6 -oy4iT42G6qk+GGn5VL5JlnJT6ZG5cqaMNFaNGlIxNb6CPUZLKq2sM3gRaimsktIw -nNAcNwQGHpe1tZo+J/Pl04JTt71Y/TTAxy7OX27aZf1Rpt0SjdZ7vTPnFDPNsHGe -KBKvPt55l2+YKjkZmV7eRevsVbpkNvNGB+T5d4Ge/wIBA6NPME0wDgYDVR0PAQH/ -BAQDAgCgMA0GA1UdDgQGBAQBAgMEMA8GA1UdIwQIMAaABAECAwQwGwYDVR0RBBQw -EoIJMTI3LjAuMC4xggVbOjoxXTALBgkqhkiG9w0BAQUDggCBAHC3gbdvc44vs+wD -g2kONiENnx8WKc0UTGg/TOXS3gaRb+CUIQtHWja65l8rAfclEovjHgZ7gx8brO0W -JuC6p3MUAKsgOssIrrRIx2rpnfcmFVMzguCmrMNVmKUAalw18Yp0F72xYAIitVQl -kJrLdIhBajcJRYu/YGltHQRaXuVt +MIIBOTCB5qADAgECAgEAMAsGCSqGSIb3DQEBBTAAMB4XDTcwMDEwMTAwMDAwMFoX +DTQ5MTIzMTIzNTk1OVowADBaMAsGCSqGSIb3DQEBAQNLADBIAkEAsuA5mAFMj6Q7 +qoBzcvKzIq4kzuT5epSp2AkcQfyBHm7K13Ws7u+0b5Vb9gqTf5cAiIKcrtrXVqkL +8i1UQF6AzwIDAQABo08wTTAOBgNVHQ8BAf8EBAMCACQwDQYDVR0OBAYEBAECAwQw +DwYDVR0jBAgwBoAEAQIDBDAbBgNVHREEFDASggkxMjcuMC4wLjGCBVs6OjFdMAsG +CSqGSIb3DQEBBQNBAJH30zjLWRztrWpOCgJL8RQWLaKzhK79pVhAx6q/3NrF16C7 ++l1BRZstTwIGdoGId8BRpErK1TXkniFb95ZMynM= -----END CERTIFICATE----- `) // localhostKey is the private key for localhostCert. var localhostKey = []byte(`-----BEGIN RSA PRIVATE KEY----- -MIIBkgIBAQKCAIB6oy4iT42G6qk+GGn5VL5JlnJT6ZG5cqaMNFaNGlIxNb6CPUZL -Kq2sM3gRaimsktIwnNAcNwQGHpe1tZo+J/Pl04JTt71Y/TTAxy7OX27aZf1Rpt0S -jdZ7vTPnFDPNsHGeKBKvPt55l2+YKjkZmV7eRevsVbpkNvNGB+T5d4Ge/wIBAwKC -AIBRwh7Bil5Z8cYpZZv7jdQxDvbim7Z7ocRdeDmzZuF2I9RW04QyHHPIIlALnBvI -YeF1veASz1gEFGUjzmbUGqKYSbCoTzXoev+F4bmbRxcX9sOmtslqvhMSHRSzA5NH -aDVI3Hn4wvBVD8gePu8ACWqvPGbCiql11OKCMfjlPn2uuwJAx/24/F5DjXZ6hQQ7 -HxScOxKrpx5WnA9r1wZTltOTZkhRRzuLc21WJeE3M15QUdWi3zZxCKRFoth65HEs -jy9YHQJAnPueRI44tz79b5QqVbeaOMUr7ZCb1Kp0uo6G+ANPLdlfliAupwij2eIz -mHRJOWk0jBtXfRft1McH2H51CpXAyw== +MIIBPQIBAAJBALLgOZgBTI+kO6qAc3LysyKuJM7k+XqUqdgJHEH8gR5uytd1rO7v +tG+VW/YKk3+XAIiCnK7a11apC/ItVEBegM8CAwEAAQJBAI5sxq7naeR9ahyqRkJi +SIv2iMxLuPEHaezf5CYOPWjSjBPyVhyRevkhtqEjF/WkgL7C2nWpYHsUcBDBQVF0 +3KECIQDtEGB2ulnkZAahl3WuJziXGLB+p8Wgx7wzSM6bHu1c6QIhAMEp++CaS+SJ +/TrU0zwY/fW4SvQeb49BPZUF3oqR8Xz3AiEA1rAJHBzBgdOQKdE3ksMUPcnvNJSN +poCcELmz2clVXtkCIQCLytuLV38XHToTipR4yMl6O+6arzAjZ56uq7m7ZRV0TwIh +AM65XAOw8Dsg9Kq78aYXiOEDc5DL0sbFUu/SlmRcCg93 -----END RSA PRIVATE KEY----- `) diff --git a/src/pkg/http/persist.go b/src/pkg/http/persist.go index e4eea6815..62f9ff1b5 100644 --- a/src/pkg/http/persist.go +++ b/src/pkg/http/persist.go @@ -111,7 +111,7 @@ func (sc *ServerConn) Read() (req *Request, err os.Error) { // Make sure body is fully consumed, even if user does not call body.Close if lastbody != nil { // body.Close is assumed to be idempotent and multiple calls to - // it should return the error that its first invokation + // it should return the error that its first invocation // returned. err = lastbody.Close() if err != nil { @@ -222,7 +222,6 @@ type ClientConn struct { pipe textproto.Pipeline writeReq func(*Request, io.Writer) os.Error - readRes func(buf *bufio.Reader, method string) (*Response, os.Error) } // NewClientConn returns a new ClientConn reading and writing c. If r is not @@ -236,7 +235,6 @@ func NewClientConn(c net.Conn, r *bufio.Reader) *ClientConn { r: r, pipereq: make(map[*Request]uint), writeReq: (*Request).Write, - readRes: ReadResponse, } } @@ -339,8 +337,13 @@ func (cc *ClientConn) Pending() int { // returned together with an ErrPersistEOF, which means that the remote // requested that this be the last request serviced. Read can be called // concurrently with Write, but not with another Read. -func (cc *ClientConn) Read(req *Request) (resp *Response, err os.Error) { +func (cc *ClientConn) Read(req *Request) (*Response, os.Error) { + return cc.readUsing(req, ReadResponse) +} +// readUsing is the implementation of Read with a replaceable +// ReadResponse-like function, used by the Transport. +func (cc *ClientConn) readUsing(req *Request, readRes func(*bufio.Reader, *Request) (*Response, os.Error)) (resp *Response, err os.Error) { // Retrieve the pipeline ID of this request/response pair cc.lk.Lock() id, ok := cc.pipereq[req] @@ -383,7 +386,7 @@ func (cc *ClientConn) Read(req *Request) (resp *Response, err os.Error) { } } - resp, err = cc.readRes(r, req.Method) + resp, err = readRes(r, req) cc.lk.Lock() defer cc.lk.Unlock() if err != nil { diff --git a/src/pkg/http/proxy_test.go b/src/pkg/http/proxy_test.go index 308bf44b4..9b320b3aa 100644 --- a/src/pkg/http/proxy_test.go +++ b/src/pkg/http/proxy_test.go @@ -40,10 +40,8 @@ func TestUseProxy(t *testing.T) { no_proxy := "foobar.com, .barbaz.net" os.Setenv("NO_PROXY", no_proxy) - tr := &Transport{} - for _, test := range UseProxyTests { - if tr.useProxy(test.host+":80") != test.match { + if useProxy(test.host+":80") != test.match { t.Errorf("useProxy(%v) = %v, want %v", test.host, !test.match, test.match) } } diff --git a/src/pkg/http/readrequest_test.go b/src/pkg/http/readrequest_test.go index 19e2ff774..d93e573f5 100644 --- a/src/pkg/http/readrequest_test.go +++ b/src/pkg/http/readrequest_test.go @@ -64,7 +64,7 @@ var reqTests = []reqTest{ Host: "www.techcrunch.com", Referer: "", UserAgent: "Fake", - Form: map[string][]string{}, + Form: Values{}, }, "abcdef\n", @@ -99,7 +99,7 @@ var reqTests = []reqTest{ Host: "test", Referer: "", UserAgent: "", - Form: map[string][]string{}, + Form: Values{}, }, "", diff --git a/src/pkg/http/request.go b/src/pkg/http/request.go index 4852ca3e1..bdc3a7e4f 100644 --- a/src/pkg/http/request.go +++ b/src/pkg/http/request.go @@ -10,8 +10,10 @@ package http import ( "bufio" + "bytes" "crypto/tls" "container/vector" + "encoding/base64" "fmt" "io" "io/ioutil" @@ -88,10 +90,10 @@ type Request struct { // // then // - // Header = map[string]string{ - // "Accept-Encoding": "gzip, deflate", - // "Accept-Language": "en-us", - // "Connection": "keep-alive", + // Header = map[string][]string{ + // "Accept-Encoding": {"gzip, deflate"}, + // "Accept-Language": {"en-us"}, + // "Connection": {"keep-alive"}, // } // // HTTP defines that header names are case-insensitive. @@ -139,7 +141,7 @@ type Request struct { UserAgent string // The parsed form. Only available after ParseForm is called. - Form map[string][]string + Form Values // The parsed multipart form, including file uploads. // Only available after ParseMultipartForm is called. @@ -230,15 +232,15 @@ const defaultUserAgent = "Go http package" // Method (defaults to "GET") // UserAgent (defaults to defaultUserAgent) // Referer -// Header +// Header (only keys not already in this list) // Cookie // ContentLength // TransferEncoding // Body // -// If Body is present but Content-Length is <= 0, Write adds -// "Transfer-Encoding: chunked" to the header. Body is closed after -// it is sent. +// If Body is present, Content-Length is <= 0 and TransferEncoding +// hasn't been set to "identity", Write adds "Transfer-Encoding: +// chunked" to the header. Body is closed after it is sent. func (req *Request) Write(w io.Writer) os.Error { return req.write(w, false) } @@ -255,6 +257,9 @@ func (req *Request) WriteProxy(w io.Writer) os.Error { func (req *Request) write(w io.Writer, usingProxy bool) os.Error { host := req.Host if host == "" { + if req.URL == nil { + return os.NewError("http: Request.Write on Request with no Host or URL set") + } host = req.URL.Host } @@ -275,9 +280,7 @@ func (req *Request) write(w io.Writer, usingProxy bool) os.Error { fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), uri) // Header lines - if !usingProxy { - fmt.Fprintf(w, "Host: %s\r\n", host) - } + fmt.Fprintf(w, "Host: %s\r\n", host) fmt.Fprintf(w, "User-Agent: %s\r\n", valueOrDefault(req.UserAgent, defaultUserAgent)) if req.Referer != "" { fmt.Fprintf(w, "Referer: %s\r\n", req.Referer) @@ -300,7 +303,7 @@ func (req *Request) write(w io.Writer, usingProxy bool) os.Error { // from Request, and introduce Request methods along the lines of // Response.{GetHeader,AddHeader} and string constants for "Host", // "User-Agent" and "Referer". - err = writeSortedHeader(w, req.Header, reqExcludeHeader) + err = req.Header.WriteSubset(w, reqExcludeHeader) if err != nil { return err } @@ -476,9 +479,35 @@ func NewRequest(method, url string, body io.Reader) (*Request, os.Error) { Body: rc, Host: u.Host, } + if body != nil { + switch v := body.(type) { + case *strings.Reader: + req.ContentLength = int64(v.Len()) + case *bytes.Buffer: + req.ContentLength = int64(v.Len()) + default: + req.ContentLength = -1 // chunked + } + if req.ContentLength == 0 { + // To prevent chunking and disambiguate this + // from the default ContentLength zero value. + req.TransferEncoding = []string{"identity"} + } + } + return req, nil } +// SetBasicAuth sets the request's Authorization header to use HTTP +// Basic Authentication with the provided username and password. +// +// With HTTP Basic Authentication the provided username and password +// are not encrypted. +func (r *Request) SetBasicAuth(username, password string) { + s := username + ":" + password + r.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(s))) +} + // ReadRequest reads and parses a request from b. func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { @@ -573,18 +602,56 @@ func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { return req, nil } +// Values maps a string key to a list of values. +// It is typically used for query parameters and form values. +// Unlike in the Header map, the keys in a Values map +// are case-sensitive. +type Values map[string][]string + +// Get gets the first value associated with the given key. +// If there are no values associated with the key, Get returns +// the empty string. To access multiple values, use the map +// directly. +func (v Values) Get(key string) string { + if v == nil { + return "" + } + vs, ok := v[key] + if !ok || len(vs) == 0 { + return "" + } + return vs[0] +} + +// Set sets the key to value. It replaces any existing +// values. +func (v Values) Set(key, value string) { + v[key] = []string{value} +} + +// Add adds the key to value. It appends to any existing +// values associated with key. +func (v Values) Add(key, value string) { + v[key] = append(v[key], value) +} + +// Del deletes the values associated with key. +func (v Values) Del(key string) { + v[key] = nil, false +} + // ParseQuery parses the URL-encoded query string and returns // a map listing the values specified for each key. // ParseQuery always returns a non-nil map containing all the // valid query parameters found; err describes the first decoding error // encountered, if any. -func ParseQuery(query string) (m map[string][]string, err os.Error) { - m = make(map[string][]string) +func ParseQuery(query string) (m Values, err os.Error) { + m = make(Values) err = parseQuery(m, query) return } -func parseQuery(m map[string][]string, query string) (err os.Error) { +func parseQuery(m Values, query string) (err os.Error) { for _, kv := range strings.Split(query, "&", -1) { if len(kv) == 0 { continue @@ -617,7 +684,7 @@ func (r *Request) ParseForm() (err os.Error) { return } - r.Form = make(map[string][]string) + r.Form = make(Values) if r.URL != nil { err = parseQuery(r.Form, r.URL.RawQuery) } diff --git a/src/pkg/http/request_test.go b/src/pkg/http/request_test.go index f79d3a242..e03ed3b05 100644 --- a/src/pkg/http/request_test.go +++ b/src/pkg/http/request_test.go @@ -162,16 +162,25 @@ func TestRedirect(t *testing.T) { defer ts.Close() var end = regexp.MustCompile("/foo/$") - r, url, err := Get(ts.URL) + r, err := Get(ts.URL) if err != nil { t.Fatal(err) } r.Body.Close() + url := r.Request.URL.String() if r.StatusCode != 200 || !end.MatchString(url) { t.Fatalf("Get got status %d at %q, want 200 matching /foo/$", r.StatusCode, url) } } +func TestSetBasicAuth(t *testing.T) { + r, _ := NewRequest("GET", "http://example.com/", nil) + r.SetBasicAuth("Aladdin", "open sesame") + if g, e := r.Header.Get("Authorization"), "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="; g != e { + t.Errorf("got header %q, want %q", g, e) + } +} + func TestMultipartRequest(t *testing.T) { // Test that we can read the values and files of a // multipart request with FormValue and FormFile, @@ -213,13 +222,13 @@ func TestEmptyMultipartRequest(t *testing.T) { func testMissingFile(t *testing.T, req *Request) { f, fh, err := req.FormFile("missing") if f != nil { - t.Errorf("FormFile file = %q, want nil", f, nil) + t.Errorf("FormFile file = %q, want nil", f) } if fh != nil { - t.Errorf("FormFile file header = %q, want nil", fh, nil) + t.Errorf("FormFile file header = %q, want nil", fh) } if err != ErrMissingFile { - t.Errorf("FormFile err = %q, want nil", err, ErrMissingFile) + t.Errorf("FormFile err = %q, want ErrMissingFile", err) } } @@ -227,7 +236,7 @@ func newTestMultipartRequest(t *testing.T) *Request { b := bytes.NewBufferString(strings.Replace(message, "\n", "\r\n", -1)) req, err := NewRequest("POST", "/", b) if err != nil { - t.Fatalf("NewRequest:", err) + t.Fatal("NewRequest:", err) } ctype := fmt.Sprintf(`multipart/form-data; boundary="%s"`, boundary) req.Header.Set("Content-type", ctype) @@ -267,7 +276,7 @@ func validateTestMultipartContents(t *testing.T, req *Request, allMem bool) { func testMultipartFile(t *testing.T, req *Request, key, expectFilename, expectContent string) multipart.File { f, fh, err := req.FormFile(key) if err != nil { - t.Fatalf("FormFile(%q):", key, err) + t.Fatalf("FormFile(%q): %q", key, err) } if fh.Filename != expectFilename { t.Errorf("filename = %q, want %q", fh.Filename, expectFilename) diff --git a/src/pkg/http/requestwrite_test.go b/src/pkg/http/requestwrite_test.go index bb000c701..98fbcf459 100644 --- a/src/pkg/http/requestwrite_test.go +++ b/src/pkg/http/requestwrite_test.go @@ -69,6 +69,7 @@ var reqWriteTests = []reqWriteTest{ "Proxy-Connection: keep-alive\r\n\r\n", "GET http://www.techcrunch.com/ HTTP/1.1\r\n" + + "Host: www.techcrunch.com\r\n" + "User-Agent: Fake\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" + "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" + @@ -101,6 +102,7 @@ var reqWriteTests = []reqWriteTest{ "6\r\nabcdef\r\n0\r\n\r\n", "GET http://www.google.com/search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + "6\r\nabcdef\r\n0\r\n\r\n", @@ -131,6 +133,7 @@ var reqWriteTests = []reqWriteTest{ "6\r\nabcdef\r\n0\r\n\r\n", "POST http://www.google.com/search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Connection: close\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + @@ -164,6 +167,7 @@ var reqWriteTests = []reqWriteTest{ "abcdef", "POST http://www.google.com/search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Connection: close\r\n" + "Content-Length: 6\r\n" + @@ -171,6 +175,35 @@ var reqWriteTests = []reqWriteTest{ "abcdef", }, + // HTTP/1.1 POST with Content-Length in headers + { + Request{ + Method: "POST", + RawURL: "http://example.com/", + Host: "example.com", + Header: Header{ + "Content-Length": []string{"10"}, // ignored + }, + ContentLength: 6, + }, + + []byte("abcdef"), + + "POST http://example.com/ HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go http package\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "abcdef", + + "POST http://example.com/ HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go http package\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "abcdef", + }, + // default to HTTP/1.1 { Request{ @@ -188,6 +221,7 @@ var reqWriteTests = []reqWriteTest{ // Looks weird but RawURL overrides what WriteProxy would choose. "GET /search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "\r\n", }, @@ -240,13 +274,45 @@ func (rc *closeChecker) Close() os.Error { // TestRequestWriteClosesBody tests that Request.Write does close its request.Body. // It also indirectly tests NewRequest and that it doesn't wrap an existing Closer -// inside a NopCloser. +// inside a NopCloser, and that it serializes it correctly. func TestRequestWriteClosesBody(t *testing.T) { rc := &closeChecker{Reader: strings.NewReader("my body")} - req, _ := NewRequest("GET", "http://foo.com/", rc) + req, _ := NewRequest("POST", "http://foo.com/", rc) + if g, e := req.ContentLength, int64(-1); g != e { + t.Errorf("got req.ContentLength %d, want %d", g, e) + } buf := new(bytes.Buffer) req.Write(buf) if !rc.closed { t.Error("body not closed after write") } + if g, e := buf.String(), "POST / HTTP/1.1\r\nHost: foo.com\r\nUser-Agent: Go http package\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nmy body\r\n0\r\n\r\n"; g != e { + t.Errorf("write:\n got: %s\nwant: %s", g, e) + } +} + +func TestZeroLengthNewRequest(t *testing.T) { + var buf bytes.Buffer + + // Writing with default identity encoding + req, _ := NewRequest("PUT", "http://foo.com/", strings.NewReader("")) + if len(req.TransferEncoding) == 0 || req.TransferEncoding[0] != "identity" { + t.Fatalf("got req.TransferEncoding of %v, want %v", req.TransferEncoding, []string{"identity"}) + } + if g, e := req.ContentLength, int64(0); g != e { + t.Errorf("got req.ContentLength %d, want %d", g, e) + } + req.Write(&buf) + if g, e := buf.String(), "PUT / HTTP/1.1\r\nHost: foo.com\r\nUser-Agent: Go http package\r\nContent-Length: 0\r\n\r\n"; g != e { + t.Errorf("identity write:\n got: %s\nwant: %s", g, e) + } + + // Overriding identity encoding and forcing chunked. + req, _ = NewRequest("PUT", "http://foo.com/", strings.NewReader("")) + req.TransferEncoding = nil + buf.Reset() + req.Write(&buf) + if g, e := buf.String(), "PUT / HTTP/1.1\r\nHost: foo.com\r\nUser-Agent: Go http package\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n"; g != e { + t.Errorf("chunked write:\n got: %s\nwant: %s", g, e) + } } diff --git a/src/pkg/http/response.go b/src/pkg/http/response.go index 1f725ecdd..42e60c1f6 100644 --- a/src/pkg/http/response.go +++ b/src/pkg/http/response.go @@ -8,11 +8,9 @@ package http import ( "bufio" - "fmt" "io" "net/textproto" "os" - "sort" "strconv" "strings" ) @@ -32,10 +30,6 @@ type Response struct { ProtoMajor int // e.g. 1 ProtoMinor int // e.g. 0 - // RequestMethod records the method used in the HTTP request. - // Header fields such as Content-Length have method-specific meaning. - RequestMethod string // e.g. "HEAD", "CONNECT", "GET", etc. - // Header maps header keys to values. If the response had multiple // headers with the same key, they will be concatenated, with comma // delimiters. (Section 4.2 of RFC 2616 requires that multiple headers @@ -70,19 +64,26 @@ type Response struct { // Trailer maps trailer keys to values, in the same // format as the header. Trailer Header + + // The Request that was sent to obtain this Response. + // Request's Body is nil (having already been consumed). + // This is only populated for Client requests. + Request *Request } -// ReadResponse reads and returns an HTTP response from r. The RequestMethod -// parameter specifies the method used in the corresponding request (e.g., -// "GET", "HEAD"). Clients must call resp.Body.Close when finished reading -// resp.Body. After that call, clients can inspect resp.Trailer to find -// key/value pairs included in the response trailer. -func ReadResponse(r *bufio.Reader, requestMethod string) (resp *Response, err os.Error) { +// ReadResponse reads and returns an HTTP response from r. The +// req parameter specifies the Request that corresponds to +// this Response. Clients must call resp.Body.Close when finished +// reading resp.Body. After that call, clients can inspect +// resp.Trailer to find key/value pairs included in the response +// trailer. +func ReadResponse(r *bufio.Reader, req *Request) (resp *Response, err os.Error) { tp := textproto.NewReader(r) resp = new(Response) - resp.RequestMethod = strings.ToUpper(requestMethod) + resp.Request = req + resp.Request.Method = strings.ToUpper(resp.Request.Method) // Parse the first line of the response. line, err := tp.ReadLine() @@ -166,7 +167,9 @@ func (r *Response) ProtoAtLeast(major, minor int) bool { func (resp *Response) Write(w io.Writer) os.Error { // RequestMethod should be upper-case - resp.RequestMethod = strings.ToUpper(resp.RequestMethod) + if resp.Request != nil { + resp.Request.Method = strings.ToUpper(resp.Request.Method) + } // Status line text := resp.Status @@ -192,7 +195,7 @@ func (resp *Response) Write(w io.Writer) os.Error { } // Rest of header - err = writeSortedHeader(w, resp.Header, respExcludeHeader) + err = resp.Header.WriteSubset(w, respExcludeHeader) if err != nil { return err } @@ -213,27 +216,3 @@ func (resp *Response) Write(w io.Writer) os.Error { // Success return nil } - -func writeSortedHeader(w io.Writer, h Header, exclude map[string]bool) os.Error { - keys := make([]string, 0, len(h)) - for k := range h { - if exclude == nil || !exclude[k] { - keys = append(keys, k) - } - } - sort.SortStrings(keys) - for _, k := range keys { - for _, v := range h[k] { - v = strings.Replace(v, "\n", " ", -1) - v = strings.Replace(v, "\r", " ", -1) - v = strings.TrimSpace(v) - if v == "" { - continue - } - if _, err := fmt.Fprintf(w, "%s: %s\r\n", k, v); err != nil { - return err - } - } - } - return nil -} diff --git a/src/pkg/http/response_test.go b/src/pkg/http/response_test.go index 9e77c20c4..1d4a23423 100644 --- a/src/pkg/http/response_test.go +++ b/src/pkg/http/response_test.go @@ -23,6 +23,10 @@ type respTest struct { Body string } +func dummyReq(method string) *Request { + return &Request{Method: method} +} + var respTests = []respTest{ // Unchunked response without Content-Length. { @@ -32,12 +36,12 @@ var respTests = []respTest{ "Body here\n", Response{ - Status: "200 OK", - StatusCode: 200, - Proto: "HTTP/1.0", - ProtoMajor: 1, - ProtoMinor: 0, - RequestMethod: "GET", + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), Header: Header{ "Connection": {"close"}, // TODO(rsc): Delete? }, @@ -61,7 +65,7 @@ var respTests = []respTest{ Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, - RequestMethod: "GET", + Request: dummyReq("GET"), Close: true, ContentLength: -1, }, @@ -81,7 +85,7 @@ var respTests = []respTest{ Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, - RequestMethod: "GET", + Request: dummyReq("GET"), Close: false, ContentLength: 0, }, @@ -98,12 +102,12 @@ var respTests = []respTest{ "Body here\n", Response{ - Status: "200 OK", - StatusCode: 200, - Proto: "HTTP/1.0", - ProtoMajor: 1, - ProtoMinor: 0, - RequestMethod: "GET", + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), Header: Header{ "Connection": {"close"}, // TODO(rsc): Delete? "Content-Length": {"10"}, // TODO(rsc): Delete? @@ -133,7 +137,7 @@ var respTests = []respTest{ Proto: "HTTP/1.0", ProtoMajor: 1, ProtoMinor: 0, - RequestMethod: "GET", + Request: dummyReq("GET"), Header: Header{}, Close: true, ContentLength: -1, @@ -160,7 +164,7 @@ var respTests = []respTest{ Proto: "HTTP/1.0", ProtoMajor: 1, ProtoMinor: 0, - RequestMethod: "GET", + Request: dummyReq("GET"), Header: Header{}, Close: true, ContentLength: -1, // TODO(rsc): Fix? @@ -183,7 +187,7 @@ var respTests = []respTest{ Proto: "HTTP/1.0", ProtoMajor: 1, ProtoMinor: 0, - RequestMethod: "HEAD", + Request: dummyReq("HEAD"), Header: Header{}, Close: true, ContentLength: 0, @@ -199,12 +203,12 @@ var respTests = []respTest{ "\r\n", Response{ - Status: "200 OK", - StatusCode: 200, - Proto: "HTTP/1.1", - ProtoMajor: 1, - ProtoMinor: 1, - RequestMethod: "GET", + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("GET"), Header: Header{ "Content-Length": {"0"}, }, @@ -225,7 +229,7 @@ var respTests = []respTest{ Proto: "HTTP/1.0", ProtoMajor: 1, ProtoMinor: 0, - RequestMethod: "GET", + Request: dummyReq("GET"), Header: Header{}, Close: true, ContentLength: -1, @@ -244,7 +248,7 @@ var respTests = []respTest{ Proto: "HTTP/1.0", ProtoMajor: 1, ProtoMinor: 0, - RequestMethod: "GET", + Request: dummyReq("GET"), Header: Header{}, Close: true, ContentLength: -1, @@ -259,7 +263,7 @@ func TestReadResponse(t *testing.T) { tt := &respTests[i] var braw bytes.Buffer braw.WriteString(tt.Raw) - resp, err := ReadResponse(bufio.NewReader(&braw), tt.Resp.RequestMethod) + resp, err := ReadResponse(bufio.NewReader(&braw), tt.Resp.Request) if err != nil { t.Errorf("#%d: %s", i, err) continue @@ -340,7 +344,7 @@ func TestReadResponseCloseInMiddle(t *testing.T) { buf.WriteString("Next Request Here") bufr := bufio.NewReader(&buf) - resp, err := ReadResponse(bufr, "GET") + resp, err := ReadResponse(bufr, dummyReq("GET")) checkErr(err, "ReadResponse") expectedLength := int64(-1) if !test.chunked { @@ -372,7 +376,7 @@ func TestReadResponseCloseInMiddle(t *testing.T) { rest, err := ioutil.ReadAll(bufr) checkErr(err, "ReadAll on remainder") if e, g := "Next Request Here", string(rest); e != g { - fatalf("for chunked=%v remainder = %q, expected %q", g, e) + fatalf("remainder = %q, expected %q", g, e) } } } @@ -381,7 +385,7 @@ func diff(t *testing.T, prefix string, have, want interface{}) { hv := reflect.ValueOf(have).Elem() wv := reflect.ValueOf(want).Elem() if hv.Type() != wv.Type() { - t.Errorf("%s: type mismatch %v vs %v", prefix, hv.Type(), wv.Type()) + t.Errorf("%s: type mismatch %v want %v", prefix, hv.Type(), wv.Type()) } for i := 0; i < hv.NumField(); i++ { hf := hv.Field(i).Interface() diff --git a/src/pkg/http/responsewrite_test.go b/src/pkg/http/responsewrite_test.go index de0635da5..f8e63acf4 100644 --- a/src/pkg/http/responsewrite_test.go +++ b/src/pkg/http/responsewrite_test.go @@ -22,7 +22,7 @@ var respWriteTests = []respWriteTest{ StatusCode: 503, ProtoMajor: 1, ProtoMinor: 0, - RequestMethod: "GET", + Request: dummyReq("GET"), Header: Header{}, Body: ioutil.NopCloser(bytes.NewBufferString("abcdef")), ContentLength: 6, @@ -38,7 +38,7 @@ var respWriteTests = []respWriteTest{ StatusCode: 200, ProtoMajor: 1, ProtoMinor: 0, - RequestMethod: "GET", + Request: dummyReq("GET"), Header: Header{}, Body: ioutil.NopCloser(bytes.NewBufferString("abcdef")), ContentLength: -1, @@ -53,7 +53,7 @@ var respWriteTests = []respWriteTest{ StatusCode: 200, ProtoMajor: 1, ProtoMinor: 1, - RequestMethod: "GET", + Request: dummyReq("GET"), Header: Header{}, Body: ioutil.NopCloser(bytes.NewBufferString("abcdef")), ContentLength: 6, @@ -71,10 +71,10 @@ var respWriteTests = []respWriteTest{ // Also tests removal of leading and trailing whitespace. { Response{ - StatusCode: 204, - ProtoMajor: 1, - ProtoMinor: 1, - RequestMethod: "GET", + StatusCode: 204, + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("GET"), Header: Header{ "Foo": []string{" Bar\nBaz "}, }, diff --git a/src/pkg/http/reverseproxy.go b/src/pkg/http/reverseproxy.go index e4ce1e34c..9a9e21599 100644 --- a/src/pkg/http/reverseproxy.go +++ b/src/pkg/http/reverseproxy.go @@ -92,6 +92,10 @@ func (p *ReverseProxy) ServeHTTP(rw ResponseWriter, req *Request) { } } + for _, cookie := range res.SetCookie { + SetCookie(rw, cookie) + } + rw.WriteHeader(res.StatusCode) if res.Body != nil { diff --git a/src/pkg/http/reverseproxy_test.go b/src/pkg/http/reverseproxy_test.go index 8cf7705d7..d7bcde90d 100644 --- a/src/pkg/http/reverseproxy_test.go +++ b/src/pkg/http/reverseproxy_test.go @@ -20,7 +20,11 @@ func TestReverseProxy(t *testing.T) { if r.Header.Get("X-Forwarded-For") == "" { t.Errorf("didn't get X-Forwarded-For header") } + if g, e := r.Host, "some-name"; g != e { + t.Errorf("backend got Host header %q, want %q", g, e) + } w.Header().Set("X-Foo", "bar") + SetCookie(w, &Cookie{Name: "flavor", Value: "chocolateChip"}) w.WriteHeader(backendStatus) w.Write([]byte(backendResponse)) })) @@ -33,7 +37,9 @@ func TestReverseProxy(t *testing.T) { frontend := httptest.NewServer(proxyHandler) defer frontend.Close() - res, _, err := Get(frontend.URL) + getReq, _ := NewRequest("GET", frontend.URL, nil) + getReq.Host = "some-name" + res, err := DefaultClient.Do(getReq) if err != nil { t.Fatalf("Get: %v", err) } @@ -43,6 +49,12 @@ func TestReverseProxy(t *testing.T) { if g, e := res.Header.Get("X-Foo"), "bar"; g != e { t.Errorf("got X-Foo %q; expected %q", g, e) } + if g, e := len(res.SetCookie), 1; g != e { + t.Fatalf("got %d SetCookies, want %d", g, e) + } + if cookie := res.SetCookie[0]; cookie.Name != "flavor" { + t.Errorf("unexpected cookie %q", cookie.Name) + } bodyBytes, _ := ioutil.ReadAll(res.Body) if g, e := string(bodyBytes), backendResponse; g != e { t.Errorf("got body %q; expected %q", g, e) diff --git a/src/pkg/http/serve_test.go b/src/pkg/http/serve_test.go index c3c7b8d33..dc4594a79 100644 --- a/src/pkg/http/serve_test.go +++ b/src/pkg/http/serve_test.go @@ -12,11 +12,14 @@ import ( "fmt" . "http" "http/httptest" + "io" "io/ioutil" + "log" "os" "net" "reflect" "strings" + "syscall" "testing" "time" ) @@ -252,7 +255,7 @@ func TestServerTimeouts(t *testing.T) { // Hit the HTTP server successfully. tr := &Transport{DisableKeepAlives: true} // they interfere with this test c := &Client{Transport: tr} - r, _, err := c.Get(url) + r, err := c.Get(url) if err != nil { t.Fatalf("http Get #1: %v", err) } @@ -282,7 +285,7 @@ func TestServerTimeouts(t *testing.T) { // Hit the HTTP server successfully again, verifying that the // previous slow connection didn't run our handler. (that we // get "req=2", not "req=3") - r, _, err = Get(url) + r, err = Get(url) if err != nil { t.Fatalf("http Get #2: %v", err) } @@ -323,7 +326,7 @@ func TestIdentityResponse(t *testing.T) { // responses. for _, te := range []string{"", "identity"} { url := ts.URL + "/?te=" + te - res, _, err := Get(url) + res, err := Get(url) if err != nil { t.Fatalf("error with Get of %s: %v", url, err) } @@ -342,7 +345,7 @@ func TestIdentityResponse(t *testing.T) { // Verify that ErrContentLength is returned url := ts.URL + "/?overwrite=1" - _, _, err := Get(url) + _, err := Get(url) if err != nil { t.Fatalf("error with Get of %s: %v", url, err) } @@ -389,7 +392,7 @@ func TestServeHTTP10Close(t *testing.T) { } r := bufio.NewReader(conn) - _, err = ReadResponse(r, "GET") + _, err = ReadResponse(r, &Request{Method: "GET"}) if err != nil { t.Fatal("ReadResponse error:", err) } @@ -417,7 +420,7 @@ func TestSetsRemoteAddr(t *testing.T) { })) defer ts.Close() - res, _, err := Get(ts.URL) + res, err := Get(ts.URL) if err != nil { t.Fatalf("Get error: %v", err) } @@ -432,13 +435,16 @@ func TestSetsRemoteAddr(t *testing.T) { } func TestChunkedResponseHeaders(t *testing.T) { + log.SetOutput(ioutil.Discard) // is noisy otherwise + defer log.SetOutput(os.Stderr) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { w.Header().Set("Content-Length", "intentional gibberish") // we check that this is deleted fmt.Fprintf(w, "I am a chunked response.") })) defer ts.Close() - res, _, err := Get(ts.URL) + res, err := Get(ts.URL) if err != nil { t.Fatalf("Get error: %v", err) } @@ -465,7 +471,7 @@ func Test304Responses(t *testing.T) { } })) defer ts.Close() - res, _, err := Get(ts.URL) + res, err := Get(ts.URL) if err != nil { t.Error(err) } @@ -490,6 +496,12 @@ func TestHeadResponses(t *testing.T) { if err != ErrBodyNotAllowed { t.Errorf("on Write, expected ErrBodyNotAllowed, got %v", err) } + + // Also exercise the ReaderFrom path + _, err = io.Copy(w, strings.NewReader("Ignored body")) + if err != ErrBodyNotAllowed { + t.Errorf("on Copy, expected ErrBodyNotAllowed, got %v", err) + } })) defer ts.Close() res, err := Head(ts.URL) @@ -516,7 +528,7 @@ func TestTLSServer(t *testing.T) { if !strings.HasPrefix(ts.URL, "https://") { t.Fatalf("expected test TLS server to start with https://, got %q", ts.URL) } - res, _, err := Get(ts.URL) + res, err := Get(ts.URL) if err != nil { t.Error(err) } @@ -551,7 +563,7 @@ var serverExpectTests = []serverExpectTest{ {100, "", true, "200 OK"}, // 100-continue but requesting client to deny us, - // so it never eads the body. + // so it never reads the body. {100, "100-continue", false, "401 Unauthorized"}, // Likewise without 100-continue: {100, "", false, "401 Unauthorized"}, @@ -618,49 +630,29 @@ func TestServerExpect(t *testing.T) { } func TestServerConsumesRequestBody(t *testing.T) { - log := make(chan string, 100) - - ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { - log <- "got_request" - w.WriteHeader(StatusOK) - log <- "wrote_header" - })) - defer ts.Close() - - conn, err := net.Dial("tcp", ts.Listener.Addr().String()) - if err != nil { - t.Fatalf("Dial: %v", err) - } - defer conn.Close() - - bufr := bufio.NewReader(conn) - gotres := make(chan bool) - go func() { - line, err := bufr.ReadString('\n') - if err != nil { - t.Fatal(err) + conn := new(testConn) + body := strings.Repeat("x", 1<<20) + conn.readBuf.Write([]byte(fmt.Sprintf( + "POST / HTTP/1.1\r\n"+ + "Host: test\r\n"+ + "Content-Length: %d\r\n"+ + "\r\n", len(body)))) + conn.readBuf.Write([]byte(body)) + + done := make(chan bool) + + ls := &oneConnListener{conn} + go Serve(ls, HandlerFunc(func(rw ResponseWriter, req *Request) { + if conn.readBuf.Len() < len(body)/2 { + t.Errorf("on request, read buffer length is %d; expected about 1MB", conn.readBuf.Len()) } - log <- line - gotres <- true - }() - - size := 1 << 20 - log <- "writing_request" - fmt.Fprintf(conn, "POST / HTTP/1.0\r\nContent-Length: %d\r\n\r\n", size) - time.Sleep(25e6) // give server chance to misbehave & speak out of turn - log <- "slept_after_req_headers" - conn.Write([]byte(strings.Repeat("a", size))) - - <-gotres - expected := []string{ - "writing_request", "got_request", - "slept_after_req_headers", "wrote_header", - "HTTP/1.0 200 OK\r\n"} - for step, e := range expected { - if g := <-log; e != g { - t.Errorf("on step %d expected %q, got %q", step, e, g) + rw.WriteHeader(200) + if g, e := conn.readBuf.Len(), 0; g != e { + t.Errorf("after WriteHeader, read buffer length is %d; want %d", g, e) } - } + done <- true + })) + <-done } func TestTimeoutHandler(t *testing.T) { @@ -677,7 +669,7 @@ func TestTimeoutHandler(t *testing.T) { // Succeed without timing out: sendHi <- true - res, _, err := Get(ts.URL) + res, err := Get(ts.URL) if err != nil { t.Error(err) } @@ -694,7 +686,7 @@ func TestTimeoutHandler(t *testing.T) { // Times out: timeout <- 1 - res, _, err = Get(ts.URL) + res, err = Get(ts.URL) if err != nil { t.Error(err) } @@ -713,3 +705,140 @@ func TestTimeoutHandler(t *testing.T) { t.Errorf("expected Write error of %v; got %v", e, g) } } + +// Verifies we don't path.Clean() on the wrong parts in redirects. +func TestRedirectMunging(t *testing.T) { + req, _ := NewRequest("GET", "http://example.com/", nil) + + resp := httptest.NewRecorder() + Redirect(resp, req, "/foo?next=http://bar.com/", 302) + if g, e := resp.Header().Get("Location"), "/foo?next=http://bar.com/"; g != e { + t.Errorf("Location header was %q; want %q", g, e) + } + + resp = httptest.NewRecorder() + Redirect(resp, req, "http://localhost:8080/_ah/login?continue=http://localhost:8080/", 302) + if g, e := resp.Header().Get("Location"), "http://localhost:8080/_ah/login?continue=http://localhost:8080/"; g != e { + t.Errorf("Location header was %q; want %q", g, e) + } +} + +// TestZeroLengthPostAndResponse exercises an optimization done by the Transport: +// when there is no body (either because the method doesn't permit a body, or an +// explicit Content-Length of zero is present), then the transport can re-use the +// connection immediately. But when it re-uses the connection, it typically closes +// the previous request's body, which is not optimal for zero-lengthed bodies, +// as the client would then see http.ErrBodyReadAfterClose and not 0, os.EOF. +func TestZeroLengthPostAndResponse(t *testing.T) { + ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, r *Request) { + all, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatalf("handler ReadAll: %v", err) + } + if len(all) != 0 { + t.Errorf("handler got %d bytes; expected 0", len(all)) + } + rw.Header().Set("Content-Length", "0") + })) + defer ts.Close() + + req, err := NewRequest("POST", ts.URL, strings.NewReader("")) + if err != nil { + t.Fatal(err) + } + req.ContentLength = 0 + + var resp [5]*Response + for i := range resp { + resp[i], err = DefaultClient.Do(req) + if err != nil { + t.Fatalf("client post #%d: %v", i, err) + } + } + + for i := range resp { + all, err := ioutil.ReadAll(resp[i].Body) + if err != nil { + t.Fatalf("req #%d: client ReadAll: %v", i, err) + } + if len(all) != 0 { + t.Errorf("req #%d: client got %d bytes; expected 0", i, len(all)) + } + } +} + +func TestHandlerPanic(t *testing.T) { + log.SetOutput(ioutil.Discard) // is noisy otherwise + defer log.SetOutput(os.Stderr) + + ts := httptest.NewServer(HandlerFunc(func(ResponseWriter, *Request) { + panic("intentional death for testing") + })) + defer ts.Close() + _, err := Get(ts.URL) + if err == nil { + t.Logf("expected an error") + } +} + +type errorListener struct { + errs []os.Error +} + +func (l *errorListener) Accept() (c net.Conn, err os.Error) { + if len(l.errs) == 0 { + return nil, os.EOF + } + err = l.errs[0] + l.errs = l.errs[1:] + return +} + +func (l *errorListener) Close() os.Error { + return nil +} + +func (l *errorListener) Addr() net.Addr { + return dummyAddr("test-address") +} + +func TestAcceptMaxFds(t *testing.T) { + log.SetOutput(ioutil.Discard) // is noisy otherwise + defer log.SetOutput(os.Stderr) + + ln := &errorListener{[]os.Error{ + &net.OpError{ + Op: "accept", + Error: os.Errno(syscall.EMFILE), + }}} + err := Serve(ln, HandlerFunc(HandlerFunc(func(ResponseWriter, *Request) {}))) + if err != os.EOF { + t.Errorf("got error %v, want EOF", err) + } +} + +func BenchmarkClientServer(b *testing.B) { + b.StopTimer() + ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, r *Request) { + fmt.Fprintf(rw, "Hello world.\n") + })) + defer ts.Close() + b.StartTimer() + + for i := 0; i < b.N; i++ { + res, err := Get(ts.URL) + if err != nil { + panic("Get: " + err.String()) + } + all, err := ioutil.ReadAll(res.Body) + if err != nil { + panic("ReadAll: " + err.String()) + } + body := string(all) + if body != "Hello world.\n" { + panic("Got body: " + body) + } + } + + b.StopTimer() +} diff --git a/src/pkg/http/server.go b/src/pkg/http/server.go index 96d2cb638..d4638f127 100644 --- a/src/pkg/http/server.go +++ b/src/pkg/http/server.go @@ -6,12 +6,12 @@ // TODO(rsc): // logging -// post support package http import ( "bufio" + "bytes" "crypto/rand" "crypto/tls" "fmt" @@ -20,6 +20,7 @@ import ( "net" "os" "path" + "runtime" "strconv" "strings" "sync" @@ -119,6 +120,27 @@ type response struct { closeAfterReply bool } +type writerOnly struct { + io.Writer +} + +func (r *response) ReadFrom(src io.Reader) (n int64, err os.Error) { + // Flush before checking r.chunking, as Flush will call + // WriteHeader if it hasn't been called yet, and WriteHeader + // is what sets r.chunking. + r.Flush() + if !r.chunking && r.bodyAllowed() { + if rf, ok := r.conn.rwc.(io.ReaderFrom); ok { + n, err = rf.ReadFrom(src) + r.written += n + return + } + } + // Fall back to default io.Copy implementation. + // Use wrapper to hide r.ReadFrom from io.Copy. + return io.Copy(writerOnly{r}, src) +} + // Create new connection from rwc. func newConn(rwc net.Conn, handler Handler) (c *conn, err os.Error) { c = new(conn) @@ -309,10 +331,19 @@ func (w *response) WriteHeader(code int) { text = "status code " + codestring } io.WriteString(w.conn.buf, proto+" "+codestring+" "+text+"\r\n") - writeSortedHeader(w.conn.buf, w.header, nil) + w.header.Write(w.conn.buf) io.WriteString(w.conn.buf, "\r\n") } +// bodyAllowed returns true if a Write is allowed for this response type. +// It's illegal to call this before the header has been flushed. +func (w *response) bodyAllowed() bool { + if !w.wroteHeader { + panic("") + } + return w.status != StatusNotModified && w.req.Method != "HEAD" +} + func (w *response) Write(data []byte) (n int, err os.Error) { if w.conn.hijacked { log.Print("http: response.Write on hijacked connection") @@ -324,9 +355,7 @@ func (w *response) Write(data []byte) (n int, err os.Error) { if len(data) == 0 { return 0, nil } - - if w.status == StatusNotModified || w.req.Method == "HEAD" { - // Must not have body. + if !w.bodyAllowed() { return 0, ErrBodyNotAllowed } @@ -454,6 +483,33 @@ func (c *conn) close() { // Serve a new connection. func (c *conn) serve() { + defer func() { + err := recover() + if err == nil { + return + } + c.rwc.Close() + + // TODO(rsc,bradfitz): this is boilerplate. move it to runtime.Stack() + var buf bytes.Buffer + fmt.Fprintf(&buf, "http: panic serving %v: %v\n", c.remoteAddr, err) + for i := 1; i < 20; i++ { + pc, file, line, ok := runtime.Caller(i) + if !ok { + break + } + var name string + f := runtime.FuncForPC(pc) + if f != nil { + name = f.Name() + } else { + name = fmt.Sprintf("%#x", pc) + } + fmt.Fprintf(&buf, " %s %s:%d\n", name, file, line) + } + log.Print(buf.String()) + }() + for { w, err := c.readRequest() if err != nil { @@ -581,12 +637,18 @@ func Redirect(w ResponseWriter, r *Request, url string, code int) { url = olddir + url } + var query string + if i := strings.Index(url, "?"); i != -1 { + url, query = url[:i], url[i:] + } + // clean up but preserve trailing slash trailing := url[len(url)-1] == '/' url = path.Clean(url) if trailing && url[len(url)-1] != '/' { url += "/" } + url += query } } @@ -805,6 +867,10 @@ func (srv *Server) Serve(l net.Listener) os.Error { for { rw, e := l.Accept() if e != nil { + if ne, ok := e.(net.Error); ok && ne.Temporary() { + log.Printf("http: Accept error: %v", e) + continue + } return e } if srv.ReadTimeout != 0 { diff --git a/src/pkg/http/spdy/Makefile b/src/pkg/http/spdy/Makefile new file mode 100644 index 000000000..3bec220c4 --- /dev/null +++ b/src/pkg/http/spdy/Makefile @@ -0,0 +1,13 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=http/spdy +GOFILES=\ + read.go\ + types.go\ + write.go\ + +include ../../../Make.pkg diff --git a/src/pkg/http/spdy/read.go b/src/pkg/http/spdy/read.go new file mode 100644 index 000000000..159dbc578 --- /dev/null +++ b/src/pkg/http/spdy/read.go @@ -0,0 +1,287 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package spdy + +import ( + "compress/zlib" + "encoding/binary" + "http" + "io" + "os" + "strings" +) + +func (frame *SynStreamFrame) read(h ControlFrameHeader, f *Framer) os.Error { + return f.readSynStreamFrame(h, frame) +} + +func (frame *SynReplyFrame) read(h ControlFrameHeader, f *Framer) os.Error { + return f.readSynReplyFrame(h, frame) +} + +func (frame *RstStreamFrame) read(h ControlFrameHeader, f *Framer) os.Error { + frame.CFHeader = h + if err := binary.Read(f.r, binary.BigEndian, &frame.StreamId); err != nil { + return err + } + if err := binary.Read(f.r, binary.BigEndian, &frame.Status); err != nil { + return err + } + return nil +} + +func (frame *SettingsFrame) read(h ControlFrameHeader, f *Framer) os.Error { + frame.CFHeader = h + var numSettings uint32 + if err := binary.Read(f.r, binary.BigEndian, &numSettings); err != nil { + return err + } + frame.FlagIdValues = make([]SettingsFlagIdValue, numSettings) + for i := uint32(0); i < numSettings; i++ { + if err := binary.Read(f.r, binary.BigEndian, &frame.FlagIdValues[i].Id); err != nil { + return err + } + frame.FlagIdValues[i].Flag = SettingsFlag((frame.FlagIdValues[i].Id & 0xff000000) >> 24) + frame.FlagIdValues[i].Id &= 0xffffff + if err := binary.Read(f.r, binary.BigEndian, &frame.FlagIdValues[i].Value); err != nil { + return err + } + } + return nil +} + +func (frame *NoopFrame) read(h ControlFrameHeader, f *Framer) os.Error { + frame.CFHeader = h + return nil +} + +func (frame *PingFrame) read(h ControlFrameHeader, f *Framer) os.Error { + frame.CFHeader = h + if err := binary.Read(f.r, binary.BigEndian, &frame.Id); err != nil { + return err + } + return nil +} + +func (frame *GoAwayFrame) read(h ControlFrameHeader, f *Framer) os.Error { + frame.CFHeader = h + if err := binary.Read(f.r, binary.BigEndian, &frame.LastGoodStreamId); err != nil { + return err + } + return nil +} + +func (frame *HeadersFrame) read(h ControlFrameHeader, f *Framer) os.Error { + return f.readHeadersFrame(h, frame) +} + +func newControlFrame(frameType ControlFrameType) (controlFrame, os.Error) { + ctor, ok := cframeCtor[frameType] + if !ok { + return nil, InvalidControlFrame + } + return ctor(), nil +} + +var cframeCtor = map[ControlFrameType]func() controlFrame{ + TypeSynStream: func() controlFrame { return new(SynStreamFrame) }, + TypeSynReply: func() controlFrame { return new(SynReplyFrame) }, + TypeRstStream: func() controlFrame { return new(RstStreamFrame) }, + TypeSettings: func() controlFrame { return new(SettingsFrame) }, + TypeNoop: func() controlFrame { return new(NoopFrame) }, + TypePing: func() controlFrame { return new(PingFrame) }, + TypeGoAway: func() controlFrame { return new(GoAwayFrame) }, + TypeHeaders: func() controlFrame { return new(HeadersFrame) }, + // TODO(willchan): Add TypeWindowUpdate +} + +type corkedReader struct { + r io.Reader + ch chan int + n int +} + +func (cr *corkedReader) Read(p []byte) (int, os.Error) { + if cr.n == 0 { + cr.n = <-cr.ch + } + if len(p) > cr.n { + p = p[:cr.n] + } + n, err := cr.r.Read(p) + cr.n -= n + return n, err +} + +func (f *Framer) uncorkHeaderDecompressor(payloadSize int) os.Error { + if f.headerDecompressor != nil { + f.headerReader.ch <- payloadSize + return nil + } + f.headerReader = corkedReader{r: f.r, ch: make(chan int, 1), n: payloadSize} + decompressor, err := zlib.NewReaderDict(&f.headerReader, []byte(HeaderDictionary)) + if err != nil { + return err + } + f.headerDecompressor = decompressor + return nil +} + +// ReadFrame reads SPDY encoded data and returns a decompressed Frame. +func (f *Framer) ReadFrame() (Frame, os.Error) { + var firstWord uint32 + if err := binary.Read(f.r, binary.BigEndian, &firstWord); err != nil { + return nil, err + } + if (firstWord & 0x80000000) != 0 { + frameType := ControlFrameType(firstWord & 0xffff) + version := uint16(0x7fff & (firstWord >> 16)) + return f.parseControlFrame(version, frameType) + } + return f.parseDataFrame(firstWord & 0x7fffffff) +} + +func (f *Framer) parseControlFrame(version uint16, frameType ControlFrameType) (Frame, os.Error) { + var length uint32 + if err := binary.Read(f.r, binary.BigEndian, &length); err != nil { + return nil, err + } + flags := ControlFlags((length & 0xff000000) >> 24) + length &= 0xffffff + header := ControlFrameHeader{version, frameType, flags, length} + cframe, err := newControlFrame(frameType) + if err != nil { + return nil, err + } + if err = cframe.read(header, f); err != nil { + return nil, err + } + return cframe, nil +} + +func parseHeaderValueBlock(r io.Reader) (http.Header, os.Error) { + var numHeaders uint16 + if err := binary.Read(r, binary.BigEndian, &numHeaders); err != nil { + return nil, err + } + h := make(http.Header, int(numHeaders)) + for i := 0; i < int(numHeaders); i++ { + var length uint16 + if err := binary.Read(r, binary.BigEndian, &length); err != nil { + return nil, err + } + nameBytes := make([]byte, length) + if _, err := io.ReadFull(r, nameBytes); err != nil { + return nil, err + } + name := string(nameBytes) + if name != strings.ToLower(name) { + return nil, UnlowercasedHeaderName + } + if h[name] != nil { + return nil, DuplicateHeaders + } + if err := binary.Read(r, binary.BigEndian, &length); err != nil { + return nil, err + } + value := make([]byte, length) + if _, err := io.ReadFull(r, value); err != nil { + return nil, err + } + valueList := strings.Split(string(value), "\x00", -1) + for _, v := range valueList { + h.Add(name, v) + } + } + return h, nil +} + +func (f *Framer) readSynStreamFrame(h ControlFrameHeader, frame *SynStreamFrame) os.Error { + frame.CFHeader = h + var err os.Error + if err = binary.Read(f.r, binary.BigEndian, &frame.StreamId); err != nil { + return err + } + if err = binary.Read(f.r, binary.BigEndian, &frame.AssociatedToStreamId); err != nil { + return err + } + if err = binary.Read(f.r, binary.BigEndian, &frame.Priority); err != nil { + return err + } + frame.Priority >>= 14 + + reader := f.r + if !f.headerCompressionDisabled { + f.uncorkHeaderDecompressor(int(h.length - 10)) + reader = f.headerDecompressor + } + + frame.Headers, err = parseHeaderValueBlock(reader) + if err != nil { + return err + } + return nil +} + +func (f *Framer) readSynReplyFrame(h ControlFrameHeader, frame *SynReplyFrame) os.Error { + frame.CFHeader = h + var err os.Error + if err = binary.Read(f.r, binary.BigEndian, &frame.StreamId); err != nil { + return err + } + var unused uint16 + if err = binary.Read(f.r, binary.BigEndian, &unused); err != nil { + return err + } + reader := f.r + if !f.headerCompressionDisabled { + f.uncorkHeaderDecompressor(int(h.length - 6)) + reader = f.headerDecompressor + } + frame.Headers, err = parseHeaderValueBlock(reader) + if err != nil { + return err + } + return nil +} + +func (f *Framer) readHeadersFrame(h ControlFrameHeader, frame *HeadersFrame) os.Error { + frame.CFHeader = h + var err os.Error + if err = binary.Read(f.r, binary.BigEndian, &frame.StreamId); err != nil { + return err + } + var unused uint16 + if err = binary.Read(f.r, binary.BigEndian, &unused); err != nil { + return err + } + reader := f.r + if !f.headerCompressionDisabled { + f.uncorkHeaderDecompressor(int(h.length - 6)) + reader = f.headerDecompressor + } + frame.Headers, err = parseHeaderValueBlock(reader) + if err != nil { + return err + } + return nil +} + +func (f *Framer) parseDataFrame(streamId uint32) (*DataFrame, os.Error) { + var length uint32 + if err := binary.Read(f.r, binary.BigEndian, &length); err != nil { + return nil, err + } + var frame DataFrame + frame.StreamId = streamId + frame.Flags = DataFlags(length >> 24) + length &= 0xffffff + frame.Data = make([]byte, length) + // TODO(willchan): Support compressed data frames. + if _, err := io.ReadFull(f.r, frame.Data); err != nil { + return nil, err + } + return &frame, nil +} diff --git a/src/pkg/http/spdy/spdy_test.go b/src/pkg/http/spdy/spdy_test.go new file mode 100644 index 000000000..9100e1ea8 --- /dev/null +++ b/src/pkg/http/spdy/spdy_test.go @@ -0,0 +1,496 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package spdy + +import ( + "bytes" + "http" + "io" + "reflect" + "testing" +) + +func TestHeaderParsing(t *testing.T) { + headers := http.Header{ + "Url": []string{"http://www.google.com/"}, + "Method": []string{"get"}, + "Version": []string{"http/1.1"}, + } + var headerValueBlockBuf bytes.Buffer + writeHeaderValueBlock(&headerValueBlockBuf, headers) + + newHeaders, err := parseHeaderValueBlock(&headerValueBlockBuf) + if err != nil { + t.Fatal("parseHeaderValueBlock:", err) + } + + if !reflect.DeepEqual(headers, newHeaders) { + t.Fatal("got: ", newHeaders, "\nwant: ", headers) + } +} + +func TestCreateParseSynStreamFrame(t *testing.T) { + buffer := new(bytes.Buffer) + framer := &Framer{ + headerCompressionDisabled: true, + w: buffer, + headerBuf: new(bytes.Buffer), + r: buffer, + } + synStreamFrame := SynStreamFrame{ + CFHeader: ControlFrameHeader{ + version: Version, + frameType: TypeSynStream, + }, + Headers: http.Header{ + "Url": []string{"http://www.google.com/"}, + "Method": []string{"get"}, + "Version": []string{"http/1.1"}, + }, + } + if err := framer.WriteFrame(&synStreamFrame); err != nil { + t.Fatal("WriteFrame without compression:", err) + } + frame, err := framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame without compression:", err) + } + parsedSynStreamFrame, ok := frame.(*SynStreamFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(synStreamFrame, *parsedSynStreamFrame) { + t.Fatal("got: ", *parsedSynStreamFrame, "\nwant: ", synStreamFrame) + } + + // Test again with compression + buffer.Reset() + framer, err = NewFramer(buffer, buffer) + if err != nil { + t.Fatal("Failed to create new framer:", err) + } + if err := framer.WriteFrame(&synStreamFrame); err != nil { + t.Fatal("WriteFrame with compression:", err) + } + frame, err = framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame with compression:", err) + } + parsedSynStreamFrame, ok = frame.(*SynStreamFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(synStreamFrame, *parsedSynStreamFrame) { + t.Fatal("got: ", *parsedSynStreamFrame, "\nwant: ", synStreamFrame) + } +} + +func TestCreateParseSynReplyFrame(t *testing.T) { + buffer := new(bytes.Buffer) + framer := &Framer{ + headerCompressionDisabled: true, + w: buffer, + headerBuf: new(bytes.Buffer), + r: buffer, + } + synReplyFrame := SynReplyFrame{ + CFHeader: ControlFrameHeader{ + version: Version, + frameType: TypeSynReply, + }, + Headers: http.Header{ + "Url": []string{"http://www.google.com/"}, + "Method": []string{"get"}, + "Version": []string{"http/1.1"}, + }, + } + if err := framer.WriteFrame(&synReplyFrame); err != nil { + t.Fatal("WriteFrame without compression:", err) + } + frame, err := framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame without compression:", err) + } + parsedSynReplyFrame, ok := frame.(*SynReplyFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(synReplyFrame, *parsedSynReplyFrame) { + t.Fatal("got: ", *parsedSynReplyFrame, "\nwant: ", synReplyFrame) + } + + // Test again with compression + buffer.Reset() + framer, err = NewFramer(buffer, buffer) + if err != nil { + t.Fatal("Failed to create new framer:", err) + } + if err := framer.WriteFrame(&synReplyFrame); err != nil { + t.Fatal("WriteFrame with compression:", err) + } + frame, err = framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame with compression:", err) + } + parsedSynReplyFrame, ok = frame.(*SynReplyFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(synReplyFrame, *parsedSynReplyFrame) { + t.Fatal("got: ", *parsedSynReplyFrame, "\nwant: ", synReplyFrame) + } +} + +func TestCreateParseRstStream(t *testing.T) { + buffer := new(bytes.Buffer) + framer, err := NewFramer(buffer, buffer) + if err != nil { + t.Fatal("Failed to create new framer:", err) + } + rstStreamFrame := RstStreamFrame{ + CFHeader: ControlFrameHeader{ + version: Version, + frameType: TypeRstStream, + }, + StreamId: 1, + Status: InvalidStream, + } + if err := framer.WriteFrame(&rstStreamFrame); err != nil { + t.Fatal("WriteFrame:", err) + } + frame, err := framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame:", err) + } + parsedRstStreamFrame, ok := frame.(*RstStreamFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(rstStreamFrame, *parsedRstStreamFrame) { + t.Fatal("got: ", *parsedRstStreamFrame, "\nwant: ", rstStreamFrame) + } +} + +func TestCreateParseSettings(t *testing.T) { + buffer := new(bytes.Buffer) + framer, err := NewFramer(buffer, buffer) + if err != nil { + t.Fatal("Failed to create new framer:", err) + } + settingsFrame := SettingsFrame{ + CFHeader: ControlFrameHeader{ + version: Version, + frameType: TypeSettings, + }, + FlagIdValues: []SettingsFlagIdValue{ + {FlagSettingsPersistValue, SettingsCurrentCwnd, 10}, + {FlagSettingsPersisted, SettingsUploadBandwidth, 1}, + }, + } + if err := framer.WriteFrame(&settingsFrame); err != nil { + t.Fatal("WriteFrame:", err) + } + frame, err := framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame:", err) + } + parsedSettingsFrame, ok := frame.(*SettingsFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(settingsFrame, *parsedSettingsFrame) { + t.Fatal("got: ", *parsedSettingsFrame, "\nwant: ", settingsFrame) + } +} + +func TestCreateParseNoop(t *testing.T) { + buffer := new(bytes.Buffer) + framer, err := NewFramer(buffer, buffer) + if err != nil { + t.Fatal("Failed to create new framer:", err) + } + noopFrame := NoopFrame{ + CFHeader: ControlFrameHeader{ + version: Version, + frameType: TypeNoop, + }, + } + if err := framer.WriteFrame(&noopFrame); err != nil { + t.Fatal("WriteFrame:", err) + } + frame, err := framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame:", err) + } + parsedNoopFrame, ok := frame.(*NoopFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(noopFrame, *parsedNoopFrame) { + t.Fatal("got: ", *parsedNoopFrame, "\nwant: ", noopFrame) + } +} + +func TestCreateParsePing(t *testing.T) { + buffer := new(bytes.Buffer) + framer, err := NewFramer(buffer, buffer) + if err != nil { + t.Fatal("Failed to create new framer:", err) + } + pingFrame := PingFrame{ + CFHeader: ControlFrameHeader{ + version: Version, + frameType: TypePing, + }, + Id: 31337, + } + if err := framer.WriteFrame(&pingFrame); err != nil { + t.Fatal("WriteFrame:", err) + } + frame, err := framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame:", err) + } + parsedPingFrame, ok := frame.(*PingFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(pingFrame, *parsedPingFrame) { + t.Fatal("got: ", *parsedPingFrame, "\nwant: ", pingFrame) + } +} + +func TestCreateParseGoAway(t *testing.T) { + buffer := new(bytes.Buffer) + framer, err := NewFramer(buffer, buffer) + if err != nil { + t.Fatal("Failed to create new framer:", err) + } + goAwayFrame := GoAwayFrame{ + CFHeader: ControlFrameHeader{ + version: Version, + frameType: TypeGoAway, + }, + LastGoodStreamId: 31337, + } + if err := framer.WriteFrame(&goAwayFrame); err != nil { + t.Fatal("WriteFrame:", err) + } + frame, err := framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame:", err) + } + parsedGoAwayFrame, ok := frame.(*GoAwayFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(goAwayFrame, *parsedGoAwayFrame) { + t.Fatal("got: ", *parsedGoAwayFrame, "\nwant: ", goAwayFrame) + } +} + +func TestCreateParseHeadersFrame(t *testing.T) { + buffer := new(bytes.Buffer) + framer := &Framer{ + headerCompressionDisabled: true, + w: buffer, + headerBuf: new(bytes.Buffer), + r: buffer, + } + headersFrame := HeadersFrame{ + CFHeader: ControlFrameHeader{ + version: Version, + frameType: TypeHeaders, + }, + } + headersFrame.Headers = http.Header{ + "Url": []string{"http://www.google.com/"}, + "Method": []string{"get"}, + "Version": []string{"http/1.1"}, + } + if err := framer.WriteFrame(&headersFrame); err != nil { + t.Fatal("WriteFrame without compression:", err) + } + frame, err := framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame without compression:", err) + } + parsedHeadersFrame, ok := frame.(*HeadersFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(headersFrame, *parsedHeadersFrame) { + t.Fatal("got: ", *parsedHeadersFrame, "\nwant: ", headersFrame) + } + + // Test again with compression + buffer.Reset() + framer, err = NewFramer(buffer, buffer) + if err := framer.WriteFrame(&headersFrame); err != nil { + t.Fatal("WriteFrame with compression:", err) + } + frame, err = framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame with compression:", err) + } + parsedHeadersFrame, ok = frame.(*HeadersFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(headersFrame, *parsedHeadersFrame) { + t.Fatal("got: ", *parsedHeadersFrame, "\nwant: ", headersFrame) + } +} + +func TestCreateParseDataFrame(t *testing.T) { + buffer := new(bytes.Buffer) + framer, err := NewFramer(buffer, buffer) + if err != nil { + t.Fatal("Failed to create new framer:", err) + } + dataFrame := DataFrame{ + StreamId: 1, + Data: []byte{'h', 'e', 'l', 'l', 'o'}, + } + if err := framer.WriteFrame(&dataFrame); err != nil { + t.Fatal("WriteFrame:", err) + } + frame, err := framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame:", err) + } + parsedDataFrame, ok := frame.(*DataFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(dataFrame, *parsedDataFrame) { + t.Fatal("got: ", *parsedDataFrame, "\nwant: ", dataFrame) + } +} + +func TestCompressionContextAcrossFrames(t *testing.T) { + buffer := new(bytes.Buffer) + framer, err := NewFramer(buffer, buffer) + if err != nil { + t.Fatal("Failed to create new framer:", err) + } + headersFrame := HeadersFrame{ + CFHeader: ControlFrameHeader{ + version: Version, + frameType: TypeHeaders, + }, + Headers: http.Header{ + "Url": []string{"http://www.google.com/"}, + "Method": []string{"get"}, + "Version": []string{"http/1.1"}, + }, + } + if err := framer.WriteFrame(&headersFrame); err != nil { + t.Fatal("WriteFrame (HEADERS):", err) + } + synStreamFrame := SynStreamFrame{ControlFrameHeader{Version, TypeSynStream, 0, 0}, 0, 0, 0, nil} + synStreamFrame.Headers = http.Header{ + "Url": []string{"http://www.google.com/"}, + "Method": []string{"get"}, + "Version": []string{"http/1.1"}, + } + if err := framer.WriteFrame(&synStreamFrame); err != nil { + t.Fatal("WriteFrame (SYN_STREAM):", err) + } + frame, err := framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame (HEADERS):", err, buffer.Bytes()) + } + parsedHeadersFrame, ok := frame.(*HeadersFrame) + if !ok { + t.Fatalf("expected HeadersFrame; got %T %v", frame, frame) + } + if !reflect.DeepEqual(headersFrame, *parsedHeadersFrame) { + t.Fatal("got: ", *parsedHeadersFrame, "\nwant: ", headersFrame) + } + frame, err = framer.ReadFrame() + if err != nil { + t.Fatal("ReadFrame (SYN_STREAM):", err, buffer.Bytes()) + } + parsedSynStreamFrame, ok := frame.(*SynStreamFrame) + if !ok { + t.Fatalf("expected SynStreamFrame; got %T %v", frame, frame) + } + if !reflect.DeepEqual(synStreamFrame, *parsedSynStreamFrame) { + t.Fatal("got: ", *parsedSynStreamFrame, "\nwant: ", synStreamFrame) + } +} + +func TestMultipleSPDYFrames(t *testing.T) { + // Initialize the framers. + pr1, pw1 := io.Pipe() + pr2, pw2 := io.Pipe() + writer, err := NewFramer(pw1, pr2) + if err != nil { + t.Fatal("Failed to create writer:", err) + } + reader, err := NewFramer(pw2, pr1) + if err != nil { + t.Fatal("Failed to create reader:", err) + } + + // Set up the frames we're actually transferring. + headersFrame := HeadersFrame{ + CFHeader: ControlFrameHeader{ + version: Version, + frameType: TypeHeaders, + }, + Headers: http.Header{ + "Url": []string{"http://www.google.com/"}, + "Method": []string{"get"}, + "Version": []string{"http/1.1"}, + }, + } + synStreamFrame := SynStreamFrame{ + CFHeader: ControlFrameHeader{ + version: Version, + frameType: TypeSynStream, + }, + Headers: http.Header{ + "Url": []string{"http://www.google.com/"}, + "Method": []string{"get"}, + "Version": []string{"http/1.1"}, + }, + } + + // Start the goroutines to write the frames. + go func() { + if err := writer.WriteFrame(&headersFrame); err != nil { + t.Fatal("WriteFrame (HEADERS): ", err) + } + if err := writer.WriteFrame(&synStreamFrame); err != nil { + t.Fatal("WriteFrame (SYN_STREAM): ", err) + } + }() + + // Read the frames and verify they look as expected. + frame, err := reader.ReadFrame() + if err != nil { + t.Fatal("ReadFrame (HEADERS): ", err) + } + parsedHeadersFrame, ok := frame.(*HeadersFrame) + if !ok { + t.Fatal("Parsed incorrect frame type:", frame) + } + if !reflect.DeepEqual(headersFrame, *parsedHeadersFrame) { + t.Fatal("got: ", *parsedHeadersFrame, "\nwant: ", headersFrame) + } + frame, err = reader.ReadFrame() + if err != nil { + t.Fatal("ReadFrame (SYN_STREAM):", err) + } + parsedSynStreamFrame, ok := frame.(*SynStreamFrame) + if !ok { + t.Fatal("Parsed incorrect frame type.") + } + if !reflect.DeepEqual(synStreamFrame, *parsedSynStreamFrame) { + t.Fatal("got: ", *parsedSynStreamFrame, "\nwant: ", synStreamFrame) + } +} diff --git a/src/pkg/http/spdy/types.go b/src/pkg/http/spdy/types.go new file mode 100644 index 000000000..5a665f04f --- /dev/null +++ b/src/pkg/http/spdy/types.go @@ -0,0 +1,363 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package spdy + +import ( + "bytes" + "compress/zlib" + "http" + "io" + "os" + "strconv" +) + +// Data Frame Format +// +----------------------------------+ +// |0| Stream-ID (31bits) | +// +----------------------------------+ +// | flags (8) | Length (24 bits) | +// +----------------------------------+ +// | Data | +// +----------------------------------+ +// +// Control Frame Format +// +----------------------------------+ +// |1| Version(15bits) | Type(16bits) | +// +----------------------------------+ +// | flags (8) | Length (24 bits) | +// +----------------------------------+ +// | Data | +// +----------------------------------+ +// +// Control Frame: SYN_STREAM +// +----------------------------------+ +// |1|000000000000001|0000000000000001| +// +----------------------------------+ +// | flags (8) | Length (24 bits) | >= 12 +// +----------------------------------+ +// |X| Stream-ID(31bits) | +// +----------------------------------+ +// |X|Associated-To-Stream-ID (31bits)| +// +----------------------------------+ +// |Pri| unused | Length (16bits)| +// +----------------------------------+ +// +// Control Frame: SYN_REPLY +// +----------------------------------+ +// |1|000000000000001|0000000000000010| +// +----------------------------------+ +// | flags (8) | Length (24 bits) | >= 8 +// +----------------------------------+ +// |X| Stream-ID(31bits) | +// +----------------------------------+ +// | unused (16 bits)| Length (16bits)| +// +----------------------------------+ +// +// Control Frame: RST_STREAM +// +----------------------------------+ +// |1|000000000000001|0000000000000011| +// +----------------------------------+ +// | flags (8) | Length (24 bits) | >= 4 +// +----------------------------------+ +// |X| Stream-ID(31bits) | +// +----------------------------------+ +// | Status code (32 bits) | +// +----------------------------------+ +// +// Control Frame: SETTINGS +// +----------------------------------+ +// |1|000000000000001|0000000000000100| +// +----------------------------------+ +// | flags (8) | Length (24 bits) | +// +----------------------------------+ +// | # of entries (32) | +// +----------------------------------+ +// +// Control Frame: NOOP +// +----------------------------------+ +// |1|000000000000001|0000000000000101| +// +----------------------------------+ +// | flags (8) | Length (24 bits) | = 0 +// +----------------------------------+ +// +// Control Frame: PING +// +----------------------------------+ +// |1|000000000000001|0000000000000110| +// +----------------------------------+ +// | flags (8) | Length (24 bits) | = 4 +// +----------------------------------+ +// | Unique id (32 bits) | +// +----------------------------------+ +// +// Control Frame: GOAWAY +// +----------------------------------+ +// |1|000000000000001|0000000000000111| +// +----------------------------------+ +// | flags (8) | Length (24 bits) | = 4 +// +----------------------------------+ +// |X| Last-accepted-stream-id | +// +----------------------------------+ +// +// Control Frame: HEADERS +// +----------------------------------+ +// |1|000000000000001|0000000000001000| +// +----------------------------------+ +// | flags (8) | Length (24 bits) | >= 8 +// +----------------------------------+ +// |X| Stream-ID (31 bits) | +// +----------------------------------+ +// | unused (16 bits)| Length (16bits)| +// +----------------------------------+ +// +// Control Frame: WINDOW_UPDATE +// +----------------------------------+ +// |1|000000000000001|0000000000001001| +// +----------------------------------+ +// | flags (8) | Length (24 bits) | = 8 +// +----------------------------------+ +// |X| Stream-ID (31 bits) | +// +----------------------------------+ +// | Delta-Window-Size (32 bits) | +// +----------------------------------+ + +// Version is the protocol version number that this package implements. +const Version = 2 + +// ControlFrameType stores the type field in a control frame header. +type ControlFrameType uint16 + +// Control frame type constants +const ( + TypeSynStream ControlFrameType = 0x0001 + TypeSynReply = 0x0002 + TypeRstStream = 0x0003 + TypeSettings = 0x0004 + TypeNoop = 0x0005 + TypePing = 0x0006 + TypeGoAway = 0x0007 + TypeHeaders = 0x0008 + TypeWindowUpdate = 0x0009 +) + +// ControlFlags are the flags that can be set on a control frame. +type ControlFlags uint8 + +const ( + ControlFlagFin ControlFlags = 0x01 +) + +// DataFlags are the flags that can be set on a data frame. +type DataFlags uint8 + +const ( + DataFlagFin DataFlags = 0x01 + DataFlagCompressed = 0x02 +) + +// MaxDataLength is the maximum number of bytes that can be stored in one frame. +const MaxDataLength = 1<<24 - 1 + +// Frame is a single SPDY frame in its unpacked in-memory representation. Use +// Framer to read and write it. +type Frame interface { + write(f *Framer) os.Error +} + +// ControlFrameHeader contains all the fields in a control frame header, +// in its unpacked in-memory representation. +type ControlFrameHeader struct { + // Note, high bit is the "Control" bit. + version uint16 + frameType ControlFrameType + Flags ControlFlags + length uint32 +} + +type controlFrame interface { + Frame + read(h ControlFrameHeader, f *Framer) os.Error +} + +// SynStreamFrame is the unpacked, in-memory representation of a SYN_STREAM +// frame. +type SynStreamFrame struct { + CFHeader ControlFrameHeader + StreamId uint32 + AssociatedToStreamId uint32 + // Note, only 2 highest bits currently used + // Rest of Priority is unused. + Priority uint16 + Headers http.Header +} + +// SynReplyFrame is the unpacked, in-memory representation of a SYN_REPLY frame. +type SynReplyFrame struct { + CFHeader ControlFrameHeader + StreamId uint32 + Headers http.Header +} + +// StatusCode represents the status that led to a RST_STREAM +type StatusCode uint32 + +const ( + ProtocolError StatusCode = 1 + InvalidStream = 2 + RefusedStream = 3 + UnsupportedVersion = 4 + Cancel = 5 + InternalError = 6 + FlowControlError = 7 +) + +// RstStreamFrame is the unpacked, in-memory representation of a RST_STREAM +// frame. +type RstStreamFrame struct { + CFHeader ControlFrameHeader + StreamId uint32 + Status StatusCode +} + +// SettingsFlag represents a flag in a SETTINGS frame. +type SettingsFlag uint8 + +const ( + FlagSettingsPersistValue SettingsFlag = 0x1 + FlagSettingsPersisted = 0x2 +) + +// SettingsFlag represents the id of an id/value pair in a SETTINGS frame. +type SettingsId uint32 + +const ( + SettingsUploadBandwidth SettingsId = 1 + SettingsDownloadBandwidth = 2 + SettingsRoundTripTime = 3 + SettingsMaxConcurrentStreams = 4 + SettingsCurrentCwnd = 5 +) + +// SettingsFlagIdValue is the unpacked, in-memory representation of the +// combined flag/id/value for a setting in a SETTINGS frame. +type SettingsFlagIdValue struct { + Flag SettingsFlag + Id SettingsId + Value uint32 +} + +// SettingsFrame is the unpacked, in-memory representation of a SPDY +// SETTINGS frame. +type SettingsFrame struct { + CFHeader ControlFrameHeader + FlagIdValues []SettingsFlagIdValue +} + +// NoopFrame is the unpacked, in-memory representation of a NOOP frame. +type NoopFrame struct { + CFHeader ControlFrameHeader +} + +// PingFrame is the unpacked, in-memory representation of a PING frame. +type PingFrame struct { + CFHeader ControlFrameHeader + Id uint32 +} + +// GoAwayFrame is the unpacked, in-memory representation of a GOAWAY frame. +type GoAwayFrame struct { + CFHeader ControlFrameHeader + LastGoodStreamId uint32 +} + +// HeadersFrame is the unpacked, in-memory representation of a HEADERS frame. +type HeadersFrame struct { + CFHeader ControlFrameHeader + StreamId uint32 + Headers http.Header +} + +// DataFrame is the unpacked, in-memory representation of a DATA frame. +type DataFrame struct { + // Note, high bit is the "Control" bit. Should be 0 for data frames. + StreamId uint32 + Flags DataFlags + Data []byte +} + +// HeaderDictionary is the dictionary sent to the zlib compressor/decompressor. +// Even though the specification states there is no null byte at the end, Chrome sends it. +const HeaderDictionary = "optionsgetheadpostputdeletetrace" + + "acceptaccept-charsetaccept-encodingaccept-languageauthorizationexpectfromhost" + + "if-modified-sinceif-matchif-none-matchif-rangeif-unmodifiedsince" + + "max-forwardsproxy-authorizationrangerefererteuser-agent" + + "100101200201202203204205206300301302303304305306307400401402403404405406407408409410411412413414415416417500501502503504505" + + "accept-rangesageetaglocationproxy-authenticatepublicretry-after" + + "servervarywarningwww-authenticateallowcontent-basecontent-encodingcache-control" + + "connectiondatetrailertransfer-encodingupgradeviawarning" + + "content-languagecontent-lengthcontent-locationcontent-md5content-rangecontent-typeetagexpireslast-modifiedset-cookie" + + "MondayTuesdayWednesdayThursdayFridaySaturdaySunday" + + "JanFebMarAprMayJunJulAugSepOctNovDec" + + "chunkedtext/htmlimage/pngimage/jpgimage/gifapplication/xmlapplication/xhtmltext/plainpublicmax-age" + + "charset=iso-8859-1utf-8gzipdeflateHTTP/1.1statusversionurl\x00" + +type FramerError int + +const ( + Internal FramerError = iota + InvalidControlFrame + UnlowercasedHeaderName + DuplicateHeaders + UnknownFrameType + InvalidDataFrame +) + +func (e FramerError) String() string { + switch e { + case Internal: + return "Internal" + case InvalidControlFrame: + return "InvalidControlFrame" + case UnlowercasedHeaderName: + return "UnlowercasedHeaderName" + case DuplicateHeaders: + return "DuplicateHeaders" + case UnknownFrameType: + return "UnknownFrameType" + case InvalidDataFrame: + return "InvalidDataFrame" + } + return "Error(" + strconv.Itoa(int(e)) + ")" +} + +// Framer handles serializing/deserializing SPDY frames, including compressing/ +// decompressing payloads. +type Framer struct { + headerCompressionDisabled bool + w io.Writer + headerBuf *bytes.Buffer + headerCompressor *zlib.Writer + r io.Reader + headerReader corkedReader + headerDecompressor io.ReadCloser +} + +// NewFramer allocates a new Framer for a given SPDY connection, repesented by +// a io.Writer and io.Reader. Note that Framer will read and write individual fields +// from/to the Reader and Writer, so the caller should pass in an appropriately +// buffered implementation to optimize performance. +func NewFramer(w io.Writer, r io.Reader) (*Framer, os.Error) { + compressBuf := new(bytes.Buffer) + compressor, err := zlib.NewWriterDict(compressBuf, zlib.BestCompression, []byte(HeaderDictionary)) + if err != nil { + return nil, err + } + framer := &Framer{ + w: w, + headerBuf: compressBuf, + headerCompressor: compressor, + r: r, + } + return framer, nil +} diff --git a/src/pkg/http/spdy/write.go b/src/pkg/http/spdy/write.go new file mode 100644 index 000000000..aa1679f1b --- /dev/null +++ b/src/pkg/http/spdy/write.go @@ -0,0 +1,287 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package spdy + +import ( + "encoding/binary" + "http" + "io" + "os" + "strings" +) + +func (frame *SynStreamFrame) write(f *Framer) os.Error { + return f.writeSynStreamFrame(frame) +} + +func (frame *SynReplyFrame) write(f *Framer) os.Error { + return f.writeSynReplyFrame(frame) +} + +func (frame *RstStreamFrame) write(f *Framer) (err os.Error) { + frame.CFHeader.version = Version + frame.CFHeader.frameType = TypeRstStream + frame.CFHeader.length = 8 + + // Serialize frame to Writer + if err = writeControlFrameHeader(f.w, frame.CFHeader); err != nil { + return + } + if err = binary.Write(f.w, binary.BigEndian, frame.StreamId); err != nil { + return + } + if err = binary.Write(f.w, binary.BigEndian, frame.Status); err != nil { + return + } + return +} + +func (frame *SettingsFrame) write(f *Framer) (err os.Error) { + frame.CFHeader.version = Version + frame.CFHeader.frameType = TypeSettings + frame.CFHeader.length = uint32(len(frame.FlagIdValues)*8 + 4) + + // Serialize frame to Writer + if err = writeControlFrameHeader(f.w, frame.CFHeader); err != nil { + return + } + if err = binary.Write(f.w, binary.BigEndian, uint32(len(frame.FlagIdValues))); err != nil { + return + } + for _, flagIdValue := range frame.FlagIdValues { + flagId := (uint32(flagIdValue.Flag) << 24) | uint32(flagIdValue.Id) + if err = binary.Write(f.w, binary.BigEndian, flagId); err != nil { + return + } + if err = binary.Write(f.w, binary.BigEndian, flagIdValue.Value); err != nil { + return + } + } + return +} + +func (frame *NoopFrame) write(f *Framer) os.Error { + frame.CFHeader.version = Version + frame.CFHeader.frameType = TypeNoop + + // Serialize frame to Writer + return writeControlFrameHeader(f.w, frame.CFHeader) +} + +func (frame *PingFrame) write(f *Framer) (err os.Error) { + frame.CFHeader.version = Version + frame.CFHeader.frameType = TypePing + frame.CFHeader.length = 4 + + // Serialize frame to Writer + if err = writeControlFrameHeader(f.w, frame.CFHeader); err != nil { + return + } + if err = binary.Write(f.w, binary.BigEndian, frame.Id); err != nil { + return + } + return +} + +func (frame *GoAwayFrame) write(f *Framer) (err os.Error) { + frame.CFHeader.version = Version + frame.CFHeader.frameType = TypeGoAway + frame.CFHeader.length = 4 + + // Serialize frame to Writer + if err = writeControlFrameHeader(f.w, frame.CFHeader); err != nil { + return + } + if err = binary.Write(f.w, binary.BigEndian, frame.LastGoodStreamId); err != nil { + return + } + return nil +} + +func (frame *HeadersFrame) write(f *Framer) os.Error { + return f.writeHeadersFrame(frame) +} + +func (frame *DataFrame) write(f *Framer) os.Error { + return f.writeDataFrame(frame) +} + +// WriteFrame writes a frame. +func (f *Framer) WriteFrame(frame Frame) os.Error { + return frame.write(f) +} + +func writeControlFrameHeader(w io.Writer, h ControlFrameHeader) os.Error { + if err := binary.Write(w, binary.BigEndian, 0x8000|h.version); err != nil { + return err + } + if err := binary.Write(w, binary.BigEndian, h.frameType); err != nil { + return err + } + flagsAndLength := (uint32(h.Flags) << 24) | h.length + if err := binary.Write(w, binary.BigEndian, flagsAndLength); err != nil { + return err + } + return nil +} + +func writeHeaderValueBlock(w io.Writer, h http.Header) (n int, err os.Error) { + n = 0 + if err = binary.Write(w, binary.BigEndian, uint16(len(h))); err != nil { + return + } + n += 2 + for name, values := range h { + if err = binary.Write(w, binary.BigEndian, uint16(len(name))); err != nil { + return + } + n += 2 + name = strings.ToLower(name) + if _, err = io.WriteString(w, name); err != nil { + return + } + n += len(name) + v := strings.Join(values, "\x00") + if err = binary.Write(w, binary.BigEndian, uint16(len(v))); err != nil { + return + } + n += 2 + if _, err = io.WriteString(w, v); err != nil { + return + } + n += len(v) + } + return +} + +func (f *Framer) writeSynStreamFrame(frame *SynStreamFrame) (err os.Error) { + // Marshal the headers. + var writer io.Writer = f.headerBuf + if !f.headerCompressionDisabled { + writer = f.headerCompressor + } + if _, err = writeHeaderValueBlock(writer, frame.Headers); err != nil { + return + } + if !f.headerCompressionDisabled { + f.headerCompressor.Flush() + } + + // Set ControlFrameHeader + frame.CFHeader.version = Version + frame.CFHeader.frameType = TypeSynStream + frame.CFHeader.length = uint32(len(f.headerBuf.Bytes()) + 10) + + // Serialize frame to Writer + if err = writeControlFrameHeader(f.w, frame.CFHeader); err != nil { + return err + } + if err = binary.Write(f.w, binary.BigEndian, frame.StreamId); err != nil { + return err + } + if err = binary.Write(f.w, binary.BigEndian, frame.AssociatedToStreamId); err != nil { + return err + } + if err = binary.Write(f.w, binary.BigEndian, frame.Priority<<14); err != nil { + return err + } + if _, err = f.w.Write(f.headerBuf.Bytes()); err != nil { + return err + } + f.headerBuf.Reset() + return nil +} + +func (f *Framer) writeSynReplyFrame(frame *SynReplyFrame) (err os.Error) { + // Marshal the headers. + var writer io.Writer = f.headerBuf + if !f.headerCompressionDisabled { + writer = f.headerCompressor + } + if _, err = writeHeaderValueBlock(writer, frame.Headers); err != nil { + return + } + if !f.headerCompressionDisabled { + f.headerCompressor.Flush() + } + + // Set ControlFrameHeader + frame.CFHeader.version = Version + frame.CFHeader.frameType = TypeSynReply + frame.CFHeader.length = uint32(len(f.headerBuf.Bytes()) + 6) + + // Serialize frame to Writer + if err = writeControlFrameHeader(f.w, frame.CFHeader); err != nil { + return + } + if err = binary.Write(f.w, binary.BigEndian, frame.StreamId); err != nil { + return + } + if err = binary.Write(f.w, binary.BigEndian, uint16(0)); err != nil { + return + } + if _, err = f.w.Write(f.headerBuf.Bytes()); err != nil { + return + } + f.headerBuf.Reset() + return +} + +func (f *Framer) writeHeadersFrame(frame *HeadersFrame) (err os.Error) { + // Marshal the headers. + var writer io.Writer = f.headerBuf + if !f.headerCompressionDisabled { + writer = f.headerCompressor + } + if _, err = writeHeaderValueBlock(writer, frame.Headers); err != nil { + return + } + if !f.headerCompressionDisabled { + f.headerCompressor.Flush() + } + + // Set ControlFrameHeader + frame.CFHeader.version = Version + frame.CFHeader.frameType = TypeHeaders + frame.CFHeader.length = uint32(len(f.headerBuf.Bytes()) + 6) + + // Serialize frame to Writer + if err = writeControlFrameHeader(f.w, frame.CFHeader); err != nil { + return + } + if err = binary.Write(f.w, binary.BigEndian, frame.StreamId); err != nil { + return + } + if err = binary.Write(f.w, binary.BigEndian, uint16(0)); err != nil { + return + } + if _, err = f.w.Write(f.headerBuf.Bytes()); err != nil { + return + } + f.headerBuf.Reset() + return +} + +func (f *Framer) writeDataFrame(frame *DataFrame) (err os.Error) { + // Validate DataFrame + if frame.StreamId&0x80000000 != 0 || len(frame.Data) >= 0x0f000000 { + return InvalidDataFrame + } + + // TODO(willchan): Support data compression. + // Serialize frame to Writer + if err = binary.Write(f.w, binary.BigEndian, frame.StreamId); err != nil { + return + } + flagsAndLength := (uint32(frame.Flags) << 24) | uint32(len(frame.Data)) + if err = binary.Write(f.w, binary.BigEndian, flagsAndLength); err != nil { + return + } + if _, err = f.w.Write(frame.Data); err != nil { + return + } + + return nil +} diff --git a/src/pkg/http/transfer.go b/src/pkg/http/transfer.go index 0fa8bed43..b54508e7a 100644 --- a/src/pkg/http/transfer.go +++ b/src/pkg/http/transfer.go @@ -38,6 +38,9 @@ func newTransferWriter(r interface{}) (t *transferWriter, err os.Error) { t.TransferEncoding = rr.TransferEncoding t.Trailer = rr.Trailer atLeastHTTP11 = rr.ProtoAtLeast(1, 1) + if t.Body != nil && t.ContentLength <= 0 && len(t.TransferEncoding) == 0 && atLeastHTTP11 { + t.TransferEncoding = []string{"chunked"} + } case *Response: t.Body = rr.Body t.ContentLength = rr.ContentLength @@ -45,7 +48,7 @@ func newTransferWriter(r interface{}) (t *transferWriter, err os.Error) { t.TransferEncoding = rr.TransferEncoding t.Trailer = rr.Trailer atLeastHTTP11 = rr.ProtoAtLeast(1, 1) - t.ResponseToHEAD = noBodyExpected(rr.RequestMethod) + t.ResponseToHEAD = noBodyExpected(rr.Request.Method) } // Sanitize Body,ContentLength,TransferEncoding @@ -95,7 +98,7 @@ func (t *transferWriter) WriteHeader(w io.Writer) (err os.Error) { if err != nil { return } - } else if t.ContentLength > 0 || t.ResponseToHEAD { + } else if t.ContentLength > 0 || t.ResponseToHEAD || (t.ContentLength == 0 && isIdentity(t.TransferEncoding)) { io.WriteString(w, "Content-Length: ") _, err = io.WriteString(w, strconv.Itoa64(t.ContentLength)+"\r\n") if err != nil { @@ -196,7 +199,7 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err os.Error) { case *Response: t.Header = rr.Header t.StatusCode = rr.StatusCode - t.RequestMethod = rr.RequestMethod + t.RequestMethod = rr.Request.Method t.ProtoMajor = rr.ProtoMajor t.ProtoMinor = rr.ProtoMinor t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header) @@ -289,6 +292,9 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err os.Error) { // Checks whether chunked is part of the encodings stack func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } +// Checks whether the encoding is explicitly "identity". +func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } + // Sanitize transfer encoding func fixTransferEncoding(requestMethod string, header Header) ([]string, os.Error) { raw, present := header["Transfer-Encoding"] diff --git a/src/pkg/http/transport.go b/src/pkg/http/transport.go index 73a2c2191..c907d85fd 100644 --- a/src/pkg/http/transport.go +++ b/src/pkg/http/transport.go @@ -6,12 +6,12 @@ package http import ( "bufio" - "bytes" "compress/gzip" "crypto/tls" "encoding/base64" "fmt" "io" + "io/ioutil" "log" "net" "os" @@ -24,7 +24,7 @@ import ( // each call to Do and uses HTTP proxies as directed by the // $HTTP_PROXY and $NO_PROXY (or $http_proxy and $no_proxy) // environment variables. -var DefaultTransport RoundTripper = &Transport{} +var DefaultTransport RoundTripper = &Transport{Proxy: ProxyFromEnvironment} // DefaultMaxIdleConnsPerHost is the default value of Transport's // MaxIdleConnsPerHost. @@ -36,12 +36,23 @@ const DefaultMaxIdleConnsPerHost = 2 type Transport struct { lk sync.Mutex idleConn map[string][]*persistConn + altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper // TODO: tunable on global max cached connections // TODO: tunable on timeout on cached connections // TODO: optional pipelining - IgnoreEnvironment bool // don't look at environment variables for proxy configuration + // Proxy specifies a function to return a proxy for a given + // Request. If the function returns a non-nil error, the + // request is aborted with the provided error. + // If Proxy is nil or returns a nil *URL, no proxy is used. + Proxy func(*Request) (*URL, os.Error) + + // Dial specifies the dial function for creating TCP + // connections. + // If Dial is nil, net.Dial is used. + Dial func(net, addr string) (c net.Conn, err os.Error) + DisableKeepAlives bool DisableCompression bool @@ -51,6 +62,39 @@ type Transport struct { MaxIdleConnsPerHost int } +// ProxyFromEnvironment returns the URL of the proxy to use for a +// given request, as indicated by the environment variables +// $HTTP_PROXY and $NO_PROXY (or $http_proxy and $no_proxy). +// Either URL or an error is returned. +func ProxyFromEnvironment(req *Request) (*URL, os.Error) { + proxy := getenvEitherCase("HTTP_PROXY") + if proxy == "" { + return nil, nil + } + if !useProxy(canonicalAddr(req.URL)) { + return nil, nil + } + proxyURL, err := ParseRequestURL(proxy) + if err != nil { + return nil, os.ErrorString("invalid proxy address") + } + if proxyURL.Host == "" { + proxyURL, err = ParseRequestURL("http://" + proxy) + if err != nil { + return nil, os.ErrorString("invalid proxy address") + } + } + return proxyURL, nil +} + +// ProxyURL returns a proxy function (for use in a Transport) +// that always returns the same URL. +func ProxyURL(url *URL) func(*Request) (*URL, os.Error) { + return func(*Request) (*URL, os.Error) { + return url, nil + } +} + // RoundTrip implements the RoundTripper interface. func (t *Transport) RoundTrip(req *Request) (resp *Response, err os.Error) { if req.URL == nil { @@ -59,7 +103,16 @@ func (t *Transport) RoundTrip(req *Request) (resp *Response, err os.Error) { } } if req.URL.Scheme != "http" && req.URL.Scheme != "https" { - return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme} + t.lk.Lock() + var rt RoundTripper + if t.altProto != nil { + rt = t.altProto[req.URL.Scheme] + } + t.lk.Unlock() + if rt == nil { + return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme} + } + return rt.RoundTrip(req) } cm, err := t.connectMethodForRequest(req) @@ -79,6 +132,27 @@ func (t *Transport) RoundTrip(req *Request) (resp *Response, err os.Error) { return pconn.roundTrip(req) } +// RegisterProtocol registers a new protocol with scheme. +// The Transport will pass requests using the given scheme to rt. +// It is rt's responsibility to simulate HTTP request semantics. +// +// RegisterProtocol can be used by other packages to provide +// implementations of protocol schemes like "ftp" or "file". +func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { + if scheme == "http" || scheme == "https" { + panic("protocol " + scheme + " already registered") + } + t.lk.Lock() + defer t.lk.Unlock() + if t.altProto == nil { + t.altProto = make(map[string]RoundTripper) + } + if _, exists := t.altProto[scheme]; exists { + panic("protocol " + scheme + " already registered") + } + t.altProto[scheme] = rt +} + // CloseIdleConnections closes any connections which were previously // connected from previous requests but are now sitting idle in // a "keep-alive" state. It does not interrupt any connections currently @@ -101,21 +175,11 @@ func (t *Transport) CloseIdleConnections() { // Private implementation past this point. // -func (t *Transport) getenvEitherCase(k string) string { - if t.IgnoreEnvironment { - return "" - } - if v := t.getenv(strings.ToUpper(k)); v != "" { +func getenvEitherCase(k string) string { + if v := os.Getenv(strings.ToUpper(k)); v != "" { return v } - return t.getenv(strings.ToLower(k)) -} - -func (t *Transport) getenv(k string) string { - if t.IgnoreEnvironment { - return "" - } - return os.Getenv(k) + return os.Getenv(strings.ToLower(k)) } func (t *Transport) connectMethodForRequest(req *Request) (*connectMethod, os.Error) { @@ -123,20 +187,12 @@ func (t *Transport) connectMethodForRequest(req *Request) (*connectMethod, os.Er targetScheme: req.URL.Scheme, targetAddr: canonicalAddr(req.URL), } - - proxy := t.getenvEitherCase("HTTP_PROXY") - if proxy != "" && t.useProxy(cm.targetAddr) { - proxyURL, err := ParseRequestURL(proxy) + if t.Proxy != nil { + var err os.Error + cm.proxyURL, err = t.Proxy(req) if err != nil { - return nil, os.ErrorString("invalid proxy address") - } - if proxyURL.Host == "" { - proxyURL, err = ParseRequestURL("http://" + proxy) - if err != nil { - return nil, os.ErrorString("invalid proxy address") - } + return nil, err } - cm.proxyURL = proxyURL } return cm, nil } @@ -149,10 +205,7 @@ func (cm *connectMethod) proxyAuth() string { } proxyInfo := cm.proxyURL.RawUserinfo if proxyInfo != "" { - enc := base64.URLEncoding - encoded := make([]byte, enc.EncodedLen(len(proxyInfo))) - enc.Encode(encoded, []byte(proxyInfo)) - return "Basic " + string(encoded) + return "Basic " + base64.URLEncoding.EncodeToString([]byte(proxyInfo)) } return "" } @@ -207,6 +260,13 @@ func (t *Transport) getIdleConn(cm *connectMethod) (pconn *persistConn) { return } +func (t *Transport) dial(network, addr string) (c net.Conn, err os.Error) { + if t.Dial != nil { + return t.Dial(network, addr) + } + return net.Dial(network, addr) +} + // getConn dials and creates a new persistConn to the target as // specified in the connectMethod. This includes doing a proxy CONNECT // and/or setting up TLS. If this doesn't return an error, the persistConn @@ -216,7 +276,7 @@ func (t *Transport) getConn(cm *connectMethod) (*persistConn, os.Error) { return pc, nil } - conn, err := net.Dial("tcp", cm.addr()) + conn, err := t.dial("tcp", cm.addr()) if err != nil { if cm.proxyURL != nil { err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err) @@ -248,18 +308,22 @@ func (t *Transport) getConn(cm *connectMethod) (*persistConn, os.Error) { } } case cm.targetScheme == "https": - fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\n", cm.targetAddr) - fmt.Fprintf(conn, "Host: %s\r\n", cm.targetAddr) + connectReq := &Request{ + Method: "CONNECT", + RawURL: cm.targetAddr, + Host: cm.targetAddr, + Header: make(Header), + } if pa != "" { - fmt.Fprintf(conn, "Proxy-Authorization: %s\r\n", pa) + connectReq.Header.Set("Proxy-Authorization", pa) } - fmt.Fprintf(conn, "\r\n") + connectReq.Write(conn) // Read response. // Okay to use and discard buffered reader here, because // TLS server will not speak until spoken to. br := bufio.NewReader(conn) - resp, err := ReadResponse(br, "CONNECT") + resp, err := ReadResponse(br, connectReq) if err != nil { conn.Close() return nil, err @@ -285,7 +349,6 @@ func (t *Transport) getConn(cm *connectMethod) (*persistConn, os.Error) { pconn.br = bufio.NewReader(pconn.conn) pconn.cc = newClientConnFunc(conn, pconn.br) - pconn.cc.readRes = readResponseWithEOFSignal go pconn.readLoop() return pconn, nil } @@ -293,7 +356,7 @@ func (t *Transport) getConn(cm *connectMethod) (*persistConn, os.Error) { // useProxy returns true if requests to addr should use a proxy, // according to the NO_PROXY or no_proxy environment variable. // addr is always a canonicalAddr with a host and port. -func (t *Transport) useProxy(addr string) bool { +func useProxy(addr string) bool { if len(addr) == 0 { return true } @@ -305,16 +368,12 @@ func (t *Transport) useProxy(addr string) bool { return false } if ip := net.ParseIP(host); ip != nil { - if ip4 := ip.To4(); ip4 != nil && ip4[0] == 127 { - // 127.0.0.0/8 loopback isn't proxied. - return false - } - if bytes.Equal(ip, net.IPv6loopback) { + if ip.IsLoopback() { return false } } - no_proxy := t.getenvEitherCase("NO_PROXY") + no_proxy := getenvEitherCase("NO_PROXY") if no_proxy == "*" { return false } @@ -447,7 +506,25 @@ func (pc *persistConn) readLoop() { } rc := <-pc.reqch - resp, err := pc.cc.Read(rc.req) + resp, err := pc.cc.readUsing(rc.req, func(buf *bufio.Reader, forReq *Request) (*Response, os.Error) { + resp, err := ReadResponse(buf, forReq) + if err != nil || resp.ContentLength == 0 { + return resp, err + } + if rc.addedGzip && resp.Header.Get("Content-Encoding") == "gzip" { + resp.Header.Del("Content-Encoding") + resp.Header.Del("Content-Length") + resp.ContentLength = -1 + gzReader, err := gzip.NewReader(resp.Body) + if err != nil { + pc.close() + return nil, err + } + resp.Body = &readFirstCloseBoth{&discardOnCloseReadCloser{gzReader}, resp.Body} + } + resp.Body = &bodyEOFSignal{body: resp.Body} + return resp, err + }) if err == ErrPersistEOF { // Succeeded, but we can't send any more @@ -469,6 +546,17 @@ func (pc *persistConn) readLoop() { waitForBodyRead <- true } } else { + // When there's no response body, we immediately + // reuse the TCP connection (putIdleConn), but + // we need to prevent ClientConn.Read from + // closing the Response.Body on the next + // loop, otherwise it might close the body + // before the client code has had a chance to + // read it (even though it'll just be 0, EOF). + pc.cc.lk.Lock() + pc.cc.lastbody = nil + pc.cc.lk.Unlock() + pc.t.putIdleConn(pc) } } @@ -491,6 +579,11 @@ type responseAndError struct { type requestAndChan struct { req *Request ch chan responseAndError + + // did the Transport (as opposed to the client code) add an + // Accept-Encoding gzip header? only if it we set it do + // we transparently decode the gzip. + addedGzip bool } func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) { @@ -522,25 +615,12 @@ func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) { } ch := make(chan responseAndError, 1) - pc.reqch <- requestAndChan{req, ch} + pc.reqch <- requestAndChan{req, ch, requestedGzip} re := <-ch pc.lk.Lock() pc.numExpectedResponses-- pc.lk.Unlock() - if re.err == nil && requestedGzip && re.res.Header.Get("Content-Encoding") == "gzip" { - re.res.Header.Del("Content-Encoding") - re.res.Header.Del("Content-Length") - re.res.ContentLength = -1 - esb := re.res.Body.(*bodyEOFSignal) - gzReader, err := gzip.NewReader(esb.body) - if err != nil { - pc.close() - return nil, err - } - esb.body = &readFirstCloseBoth{gzReader, esb.body} - } - return re.res, re.err } @@ -572,16 +652,6 @@ func responseIsKeepAlive(res *Response) bool { return false } -// readResponseWithEOFSignal is a wrapper around ReadResponse that replaces -// the response body with a bodyEOFSignal-wrapped version. -func readResponseWithEOFSignal(r *bufio.Reader, requestMethod string) (resp *Response, err os.Error) { - resp, err = ReadResponse(r, requestMethod) - if err == nil && resp.ContentLength != 0 { - resp.Body = &bodyEOFSignal{body: resp.Body} - } - return -} - // bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most // once, right before the final Read() or Close() call returns, but after // EOF has been seen. @@ -604,6 +674,9 @@ func (es *bodyEOFSignal) Read(p []byte) (n int, err os.Error) { } func (es *bodyEOFSignal) Close() (err os.Error) { + if es.isClosed { + return nil + } es.isClosed = true err = es.body.Close() if err == nil && es.fn != nil { @@ -628,3 +701,13 @@ func (r *readFirstCloseBoth) Close() os.Error { } return nil } + +// discardOnCloseReadCloser consumes all its input on Close. +type discardOnCloseReadCloser struct { + io.ReadCloser +} + +func (d *discardOnCloseReadCloser) Close() os.Error { + io.Copy(ioutil.Discard, d.ReadCloser) // ignore errors; likely invalid or already closed + return d.ReadCloser.Close() +} diff --git a/src/pkg/http/transport_test.go b/src/pkg/http/transport_test.go index a32ac4c4f..76e97640e 100644 --- a/src/pkg/http/transport_test.go +++ b/src/pkg/http/transport_test.go @@ -17,6 +17,7 @@ import ( "io/ioutil" "os" "strconv" + "strings" "testing" "time" ) @@ -43,7 +44,7 @@ func TestTransportKeepAlives(t *testing.T) { c := &Client{Transport: tr} fetch := func(n int) string { - res, _, err := c.Get(ts.URL) + res, err := c.Get(ts.URL) if err != nil { t.Fatalf("error in disableKeepAlive=%v, req #%d, GET: %v", disableKeepAlive, n, err) } @@ -160,7 +161,7 @@ func TestTransportIdleCacheKeys(t *testing.T) { t.Errorf("After CloseIdleConnections expected %d idle conn cache keys; got %d", e, g) } - resp, _, err := c.Get(ts.URL) + resp, err := c.Get(ts.URL) if err != nil { t.Error(err) } @@ -201,7 +202,7 @@ func TestTransportMaxPerHostIdleConns(t *testing.T) { // Their responses will hang until we we write to resch, though. donech := make(chan bool) doReq := func() { - resp, _, err := c.Get(ts.URL) + resp, err := c.Get(ts.URL) if err != nil { t.Error(err) } @@ -256,26 +257,44 @@ func TestTransportServerClosingUnexpectedly(t *testing.T) { tr := &Transport{} c := &Client{Transport: tr} - fetch := func(n int) string { - res, _, err := c.Get(ts.URL) - if err != nil { - t.Fatalf("error in req #%d, GET: %v", n, err) + fetch := func(n, retries int) string { + condFatalf := func(format string, arg ...interface{}) { + if retries <= 0 { + t.Fatalf(format, arg...) + } + t.Logf("retrying shortly after expected error: "+format, arg...) + time.Sleep(1e9 / int64(retries)) } - body, err := ioutil.ReadAll(res.Body) - if err != nil { - t.Fatalf("error in req #%d, ReadAll: %v", n, err) + for retries >= 0 { + retries-- + res, err := c.Get(ts.URL) + if err != nil { + condFatalf("error in req #%d, GET: %v", n, err) + continue + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + condFatalf("error in req #%d, ReadAll: %v", n, err) + continue + } + res.Body.Close() + return string(body) } - res.Body.Close() - return string(body) + panic("unreachable") } - body1 := fetch(1) - body2 := fetch(2) + body1 := fetch(1, 0) + body2 := fetch(2, 0) ts.CloseClientConnections() // surprise! - time.Sleep(25e6) // idle for a bit (test is inherently racey, but expectedly) - body3 := fetch(3) + // This test has an expected race. Sleeping for 25 ms prevents + // it on most fast machines, causing the next fetch() call to + // succeed quickly. But if we do get errors, fetch() will retry 5 + // times with some delays between. + time.Sleep(25e6) + + body3 := fetch(3, 5) if body1 != body2 { t.Errorf("expected body1 and body2 to be equal") @@ -376,6 +395,9 @@ func TestTransportGzip(t *testing.T) { t.Errorf("Accept-Encoding = %q, want %q", g, e) } rw.Header().Set("Content-Encoding", "gzip") + if req.Method == "HEAD" { + return + } var w io.Writer = rw var buf bytes.Buffer @@ -399,7 +421,7 @@ func TestTransportGzip(t *testing.T) { c := &Client{Transport: &Transport{}} // First fetch something large, but only read some of it. - res, _, err := c.Get(ts.URL + "?body=large&chunked=" + chunked) + res, err := c.Get(ts.URL + "?body=large&chunked=" + chunked) if err != nil { t.Fatalf("large get: %v", err) } @@ -419,7 +441,7 @@ func TestTransportGzip(t *testing.T) { } // Then something small. - res, _, err = c.Get(ts.URL + "?chunked=" + chunked) + res, err = c.Get(ts.URL + "?chunked=" + chunked) if err != nil { t.Fatal(err) } @@ -445,6 +467,40 @@ func TestTransportGzip(t *testing.T) { t.Errorf("expected Read error after Close; got %d, %v", n, err) } } + + // And a HEAD request too, because they're always weird. + c := &Client{Transport: &Transport{}} + res, err := c.Head(ts.URL) + if err != nil { + t.Fatalf("Head: %v", err) + } + if res.StatusCode != 200 { + t.Errorf("Head status=%d; want=200", res.StatusCode) + } +} + +func TestTransportProxy(t *testing.T) { + ch := make(chan string, 1) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + ch <- "real server" + })) + defer ts.Close() + proxy := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + ch <- "proxy for " + r.URL.String() + })) + defer proxy.Close() + + pu, err := ParseURL(proxy.URL) + if err != nil { + t.Fatal(err) + } + c := &Client{Transport: &Transport{Proxy: ProxyURL(pu)}} + c.Head(ts.URL) + got := <-ch + want := "proxy for " + ts.URL + "/" + if got != want { + t.Errorf("want %q, got %q", want, got) + } } // TestTransportGzipRecursive sends a gzip quine and checks that the @@ -459,7 +515,7 @@ func TestTransportGzipRecursive(t *testing.T) { defer ts.Close() c := &Client{Transport: &Transport{}} - res, _, err := c.Get(ts.URL) + res, err := c.Get(ts.URL) if err != nil { t.Fatal(err) } @@ -476,6 +532,36 @@ func TestTransportGzipRecursive(t *testing.T) { } } +type fooProto struct{} + +func (fooProto) RoundTrip(req *Request) (*Response, os.Error) { + res := &Response{ + Status: "200 OK", + StatusCode: 200, + Header: make(Header), + Body: ioutil.NopCloser(strings.NewReader("You wanted " + req.URL.String())), + } + return res, nil +} + +func TestTransportAltProto(t *testing.T) { + tr := &Transport{} + c := &Client{Transport: tr} + tr.RegisterProtocol("foo", fooProto{}) + res, err := c.Get("foo://bar.com/path") + if err != nil { + t.Fatal(err) + } + bodyb, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + body := string(bodyb) + if e := "You wanted foo://bar.com/path"; body != e { + t.Errorf("got response %q, want %q", body, e) + } +} + // rgz is a gzip quine that uncompresses to itself. var rgz = []byte{ 0x1f, 0x8b, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, diff --git a/src/pkg/http/url.go b/src/pkg/http/url.go index 0fc0cb2d7..05b1662d3 100644 --- a/src/pkg/http/url.go +++ b/src/pkg/http/url.go @@ -449,7 +449,7 @@ func ParseURLReference(rawurlref string) (url *URL, err os.Error) { // // There are redundant fields stored in the URL structure: // the String method consults Scheme, Path, Host, RawUserinfo, -// RawQuery, and Fragment, but not Raw, RawPath or Authority. +// RawQuery, and Fragment, but not Raw, RawPath or RawAuthority. func (url *URL) String() string { result := "" if url.Scheme != "" { @@ -486,10 +486,14 @@ func (url *URL) String() string { return result } -// EncodeQuery encodes the query represented as a multimap. -func EncodeQuery(m map[string][]string) string { - parts := make([]string, 0, len(m)) // will be large enough for most uses - for k, vs := range m { +// Encode encodes the values into ``URL encoded'' form. +// e.g. "foo=bar&bar=baz" +func (v Values) Encode() string { + if v == nil { + return "" + } + parts := make([]string, 0, len(v)) // will be large enough for most uses + for k, vs := range v { prefix := URLEscape(k) + "=" for _, v := range vs { parts = append(parts, prefix+URLEscape(v)) @@ -593,3 +597,9 @@ func (base *URL) ResolveReference(ref *URL) *URL { url.Raw = url.String() return url } + +// Query parses RawQuery and returns the corresponding values. +func (u *URL) Query() Values { + v, _ := ParseQuery(u.RawQuery) + return v +} diff --git a/src/pkg/http/url_test.go b/src/pkg/http/url_test.go index d8863f3d3..eaec5872a 100644 --- a/src/pkg/http/url_test.go +++ b/src/pkg/http/url_test.go @@ -538,23 +538,21 @@ func TestUnescapeUserinfo(t *testing.T) { } } -type qMap map[string][]string - type EncodeQueryTest struct { - m qMap + m Values expected string expected1 string } var encodeQueryTests = []EncodeQueryTest{ {nil, "", ""}, - {qMap{"q": {"puppies"}, "oe": {"utf8"}}, "q=puppies&oe=utf8", "oe=utf8&q=puppies"}, - {qMap{"q": {"dogs", "&", "7"}}, "q=dogs&q=%26&q=7", "q=dogs&q=%26&q=7"}, + {Values{"q": {"puppies"}, "oe": {"utf8"}}, "q=puppies&oe=utf8", "oe=utf8&q=puppies"}, + {Values{"q": {"dogs", "&", "7"}}, "q=dogs&q=%26&q=7", "q=dogs&q=%26&q=7"}, } func TestEncodeQuery(t *testing.T) { for _, tt := range encodeQueryTests { - if q := EncodeQuery(tt.m); q != tt.expected && q != tt.expected1 { + if q := tt.m.Encode(); q != tt.expected && q != tt.expected1 { t.Errorf(`EncodeQuery(%+v) = %q, want %q`, tt.m, q, tt.expected) } } @@ -673,3 +671,28 @@ func TestResolveReference(t *testing.T) { } } + +func TestQueryValues(t *testing.T) { + u, _ := ParseURL("http://x.com?foo=bar&bar=1&bar=2") + v := u.Query() + if len(v) != 2 { + t.Errorf("got %d keys in Query values, want 2", len(v)) + } + if g, e := v.Get("foo"), "bar"; g != e { + t.Errorf("Get(foo) = %q, want %q", g, e) + } + // Case sensitive: + if g, e := v.Get("Foo"), ""; g != e { + t.Errorf("Get(Foo) = %q, want %q", g, e) + } + if g, e := v.Get("bar"), "1"; g != e { + t.Errorf("Get(bar) = %q, want %q", g, e) + } + if g, e := v.Get("baz"), ""; g != e { + t.Errorf("Get(baz) = %q, want %q", g, e) + } + v.Del("bar") + if g, e := v.Get("bar"), ""; g != e { + t.Errorf("second Get(bar) = %q, want %q", g, e) + } +} diff --git a/src/pkg/image/bmp/Makefile b/src/pkg/image/bmp/Makefile new file mode 100644 index 000000000..56635f7ce --- /dev/null +++ b/src/pkg/image/bmp/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=image/bmp +GOFILES=\ + reader.go\ + +include ../../../Make.pkg diff --git a/src/pkg/image/bmp/reader.go b/src/pkg/image/bmp/reader.go new file mode 100644 index 000000000..f2842caed --- /dev/null +++ b/src/pkg/image/bmp/reader.go @@ -0,0 +1,148 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bmp implements a BMP image decoder. +// +// The BMP specification is at http://www.digicamsoft.com/bmp/bmp.html. +package bmp + +import ( + "image" + "io" + "os" +) + +// ErrUnsupported means that the input BMP image uses a valid but unsupported +// feature. +var ErrUnsupported = os.NewError("bmp: unsupported BMP image") + +func readUint16(b []byte) uint16 { + return uint16(b[0]) | uint16(b[1])<<8 +} + +func readUint32(b []byte) uint32 { + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 +} + +// decodePaletted reads an 8 bit-per-pixel BMP image from r. +func decodePaletted(r io.Reader, c image.Config) (image.Image, os.Error) { + var tmp [4]byte + paletted := image.NewPaletted(c.Width, c.Height, c.ColorModel.(image.PalettedColorModel)) + // BMP images are stored bottom-up rather than top-down. + for y := c.Height - 1; y >= 0; y-- { + p := paletted.Pix[y*paletted.Stride : y*paletted.Stride+c.Width] + _, err := io.ReadFull(r, p) + if err != nil { + return nil, err + } + // Each row is 4-byte aligned. + if c.Width%4 != 0 { + _, err := io.ReadFull(r, tmp[:4-c.Width%4]) + if err != nil { + return nil, err + } + } + } + return paletted, nil +} + +// decodeRGBA reads a 24 bit-per-pixel BMP image from r. +func decodeRGBA(r io.Reader, c image.Config) (image.Image, os.Error) { + rgba := image.NewRGBA(c.Width, c.Height) + // There are 3 bytes per pixel, and each row is 4-byte aligned. + b := make([]byte, (3*c.Width+3)&^3) + // BMP images are stored bottom-up rather than top-down. + for y := c.Height - 1; y >= 0; y-- { + _, err := io.ReadFull(r, b) + if err != nil { + return nil, err + } + p := rgba.Pix[y*rgba.Stride : y*rgba.Stride+c.Width] + for x := range p { + // BMP images are stored in BGR order rather than RGB order. + p[x] = image.RGBAColor{b[3*x+2], b[3*x+1], b[3*x+0], 0xFF} + } + } + return rgba, nil +} + +// Decode reads a BMP image from r and returns it as an image.Image. +// Limitation: The file must be 8 or 24 bits per pixel. +func Decode(r io.Reader) (image.Image, os.Error) { + c, err := DecodeConfig(r) + if err != nil { + return nil, err + } + if c.ColorModel == image.RGBAColorModel { + return decodeRGBA(r, c) + } + return decodePaletted(r, c) +} + +// DecodeConfig returns the color model and dimensions of a BMP image without +// decoding the entire image. +// Limitation: The file must be 8 or 24 bits per pixel. +func DecodeConfig(r io.Reader) (config image.Config, err os.Error) { + // We only support those BMP images that are a BITMAPFILEHEADER + // immediately followed by a BITMAPINFOHEADER. + const ( + fileHeaderLen = 14 + infoHeaderLen = 40 + ) + var b [1024]byte + if _, err = io.ReadFull(r, b[:fileHeaderLen+infoHeaderLen]); err != nil { + return + } + if string(b[:2]) != "BM" { + err = os.NewError("bmp: invalid format") + return + } + offset := readUint32(b[10:14]) + if readUint32(b[14:18]) != infoHeaderLen { + err = ErrUnsupported + return + } + width := int(readUint32(b[18:22])) + height := int(readUint32(b[22:26])) + if width < 0 || height < 0 { + err = ErrUnsupported + return + } + // We only support 1 plane, 8 or 24 bits per pixel and no compression. + planes, bpp, compression := readUint16(b[26:28]), readUint16(b[28:30]), readUint32(b[30:34]) + if planes != 1 || compression != 0 { + err = ErrUnsupported + return + } + switch bpp { + case 8: + if offset != fileHeaderLen+infoHeaderLen+256*4 { + err = ErrUnsupported + return + } + _, err = io.ReadFull(r, b[:256*4]) + if err != nil { + return + } + pcm := make(image.PalettedColorModel, 256) + for i := range pcm { + // BMP images are stored in BGR order rather than RGB order. + // Every 4th byte is padding. + pcm[i] = image.RGBAColor{b[4*i+2], b[4*i+1], b[4*i+0], 0xFF} + } + return image.Config{pcm, width, height}, nil + case 24: + if offset != fileHeaderLen+infoHeaderLen { + err = ErrUnsupported + return + } + return image.Config{image.RGBAColorModel, width, height}, nil + } + err = ErrUnsupported + return +} + +func init() { + image.RegisterFormat("bmp", "BM????\x00\x00\x00\x00", Decode, DecodeConfig) +} diff --git a/src/pkg/image/decode_test.go b/src/pkg/image/decode_test.go index 0716ad905..540d5eda5 100644 --- a/src/pkg/image/decode_test.go +++ b/src/pkg/image/decode_test.go @@ -10,27 +10,34 @@ import ( "os" "testing" - // TODO(nigeltao): implement bmp, gif and tiff decoders. + _ "image/bmp" + _ "image/gif" _ "image/jpeg" _ "image/png" + _ "image/tiff" ) -const goldenFile = "testdata/video-001.png" - type imageTest struct { - filename string - tolerance int + goldenFilename string + filename string + tolerance int } var imageTests = []imageTest{ - //{"testdata/video-001.bmp", 0}, + {"testdata/video-001.png", "testdata/video-001.bmp", 0}, // GIF images are restricted to a 256-color palette and the conversion // to GIF loses significant image quality. - //{"testdata/video-001.gif", 64<<8}, + {"testdata/video-001.png", "testdata/video-001.gif", 64 << 8}, + {"testdata/video-001.png", "testdata/video-001.interlaced.gif", 64 << 8}, + {"testdata/video-001.png", "testdata/video-001.5bpp.gif", 128 << 8}, // JPEG is a lossy format and hence needs a non-zero tolerance. - {"testdata/video-001.jpeg", 8 << 8}, - {"testdata/video-001.png", 0}, - //{"testdata/video-001.tiff", 0}, + {"testdata/video-001.png", "testdata/video-001.jpeg", 8 << 8}, + {"testdata/video-001.png", "testdata/video-001.png", 0}, + {"testdata/video-001.png", "testdata/video-001.tiff", 0}, + + // Test grayscale images. + {"testdata/video-005.gray.png", "testdata/video-005.gray.jpeg", 8 << 8}, + {"testdata/video-005.gray.png", "testdata/video-005.gray.png", 0}, } func decode(filename string) (image.Image, string, os.Error) { @@ -42,6 +49,15 @@ func decode(filename string) (image.Image, string, os.Error) { return image.Decode(bufio.NewReader(f)) } +func decodeConfig(filename string) (image.Config, string, os.Error) { + f, err := os.Open(filename) + if err != nil { + return image.Config{}, "", err + } + defer f.Close() + return image.DecodeConfig(bufio.NewReader(f)) +} + func delta(u0, u1 uint32) int { d := int(u0) - int(u1) if d < 0 { @@ -61,29 +77,47 @@ func withinTolerance(c0, c1 image.Color, tolerance int) bool { } func TestDecode(t *testing.T) { - golden, _, err := decode(goldenFile) - if err != nil { - t.Errorf("%s: %v", goldenFile, err) - } + golden := make(map[string]image.Image) loop: for _, it := range imageTests { - m, _, err := decode(it.filename) + g := golden[it.goldenFilename] + if g == nil { + var err os.Error + g, _, err = decode(it.goldenFilename) + if err != nil { + t.Errorf("%s: %v", it.goldenFilename, err) + continue loop + } + golden[it.goldenFilename] = g + } + m, imageFormat, err := decode(it.filename) if err != nil { t.Errorf("%s: %v", it.filename, err) continue loop } - b := golden.Bounds() + b := g.Bounds() if !b.Eq(m.Bounds()) { t.Errorf("%s: want bounds %v got %v", it.filename, b, m.Bounds()) continue loop } for y := b.Min.Y; y < b.Max.Y; y++ { for x := b.Min.X; x < b.Max.X; x++ { - if !withinTolerance(golden.At(x, y), m.At(x, y), it.tolerance) { - t.Errorf("%s: at (%d, %d), want %v got %v", it.filename, x, y, golden.At(x, y), m.At(x, y)) + if !withinTolerance(g.At(x, y), m.At(x, y), it.tolerance) { + t.Errorf("%s: at (%d, %d), want %v got %v", it.filename, x, y, g.At(x, y), m.At(x, y)) continue loop } } } + if imageFormat == "gif" { + // Each frame of a GIF can have a frame-local palette override the + // GIF-global palette. Thus, image.Decode can yield a different ColorModel + // than image.DecodeConfig. + continue + } + c, _, err := decodeConfig(it.filename) + if m.ColorModel() != c.ColorModel { + t.Errorf("%s: color models differ", it.filename) + continue loop + } } } diff --git a/src/pkg/exp/draw/Makefile b/src/pkg/image/draw/Makefile index 6f0f0b2f5..2ba6e7b51 100644 --- a/src/pkg/exp/draw/Makefile +++ b/src/pkg/image/draw/Makefile @@ -4,9 +4,8 @@ include ../../../Make.inc -TARG=exp/draw +TARG=image/draw GOFILES=\ draw.go\ - event.go\ include ../../../Make.pkg diff --git a/src/pkg/image/draw/clip_test.go b/src/pkg/image/draw/clip_test.go new file mode 100644 index 000000000..db40d82f5 --- /dev/null +++ b/src/pkg/image/draw/clip_test.go @@ -0,0 +1,193 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package draw + +import ( + "image" + "testing" +) + +type clipTest struct { + desc string + r, dr, sr, mr image.Rectangle + sp, mp image.Point + nilMask bool + r0 image.Rectangle + sp0, mp0 image.Point +} + +var clipTests = []clipTest{ + // The following tests all have a nil mask. + { + "basic", + image.Rect(0, 0, 100, 100), + image.Rect(0, 0, 100, 100), + image.Rect(0, 0, 100, 100), + image.ZR, + image.ZP, + image.ZP, + true, + image.Rect(0, 0, 100, 100), + image.ZP, + image.ZP, + }, + { + "clip dr", + image.Rect(0, 0, 100, 100), + image.Rect(40, 40, 60, 60), + image.Rect(0, 0, 100, 100), + image.ZR, + image.ZP, + image.ZP, + true, + image.Rect(40, 40, 60, 60), + image.Pt(40, 40), + image.ZP, + }, + { + "clip sr", + image.Rect(0, 0, 100, 100), + image.Rect(0, 0, 100, 100), + image.Rect(20, 20, 80, 80), + image.ZR, + image.ZP, + image.ZP, + true, + image.Rect(20, 20, 80, 80), + image.Pt(20, 20), + image.ZP, + }, + { + "clip dr and sr", + image.Rect(0, 0, 100, 100), + image.Rect(0, 0, 50, 100), + image.Rect(20, 20, 80, 80), + image.ZR, + image.ZP, + image.ZP, + true, + image.Rect(20, 20, 50, 80), + image.Pt(20, 20), + image.ZP, + }, + { + "clip dr and sr, sp outside sr (top-left)", + image.Rect(0, 0, 100, 100), + image.Rect(0, 0, 50, 100), + image.Rect(20, 20, 80, 80), + image.ZR, + image.Pt(15, 8), + image.ZP, + true, + image.Rect(5, 12, 50, 72), + image.Pt(20, 20), + image.ZP, + }, + { + "clip dr and sr, sp outside sr (middle-left)", + image.Rect(0, 0, 100, 100), + image.Rect(0, 0, 50, 100), + image.Rect(20, 20, 80, 80), + image.ZR, + image.Pt(15, 66), + image.ZP, + true, + image.Rect(5, 0, 50, 14), + image.Pt(20, 66), + image.ZP, + }, + { + "clip dr and sr, sp outside sr (bottom-left)", + image.Rect(0, 0, 100, 100), + image.Rect(0, 0, 50, 100), + image.Rect(20, 20, 80, 80), + image.ZR, + image.Pt(15, 91), + image.ZP, + true, + image.ZR, + image.Pt(15, 91), + image.ZP, + }, + { + "clip dr and sr, sp inside sr", + image.Rect(0, 0, 100, 100), + image.Rect(0, 0, 50, 100), + image.Rect(20, 20, 80, 80), + image.ZR, + image.Pt(44, 33), + image.ZP, + true, + image.Rect(0, 0, 36, 47), + image.Pt(44, 33), + image.ZP, + }, + + // The following tests all have a non-nil mask. + { + "basic mask", + image.Rect(0, 0, 80, 80), + image.Rect(20, 0, 100, 80), + image.Rect(0, 0, 50, 49), + image.Rect(0, 0, 46, 47), + image.ZP, + image.ZP, + false, + image.Rect(20, 0, 46, 47), + image.Pt(20, 0), + image.Pt(20, 0), + }, + // TODO(nigeltao): write more tests. +} + +func TestClip(t *testing.T) { + dst0 := image.NewRGBA(100, 100) + src0 := image.NewRGBA(100, 100) + mask0 := image.NewRGBA(100, 100) + for _, c := range clipTests { + dst := dst0.SubImage(c.dr).(*image.RGBA) + src := src0.SubImage(c.sr).(*image.RGBA) + var mask image.Image + if !c.nilMask { + mask = mask0.SubImage(c.mr) + } + r, sp, mp := c.r, c.sp, c.mp + clip(dst, &r, src, &sp, mask, &mp) + + // Check that the actual results equal the expected results. + if !c.r0.Eq(r) { + t.Errorf("%s: clip rectangle want %v got %v", c.desc, c.r0, r) + continue + } + if !c.sp0.Eq(sp) { + t.Errorf("%s: sp want %v got %v", c.desc, c.sp0, sp) + continue + } + if !c.nilMask { + if !c.mp0.Eq(mp) { + t.Errorf("%s: mp want %v got %v", c.desc, c.mp0, mp) + continue + } + } + + // Check that the clipped rectangle is contained by the dst / src / mask + // rectangles, in their respective co-ordinate spaces. + if !r.In(c.dr) { + t.Errorf("%s: c.dr %v does not contain r %v", c.desc, c.dr, r) + } + // sr is r translated into src's co-ordinate space. + sr := r.Add(c.sp.Sub(c.dr.Min)) + if !sr.In(c.sr) { + t.Errorf("%s: c.sr %v does not contain sr %v", c.desc, c.sr, sr) + } + if !c.nilMask { + // mr is r translated into mask's co-ordinate space. + mr := r.Add(c.mp.Sub(c.dr.Min)) + if !mr.In(c.mr) { + t.Errorf("%s: c.mr %v does not contain mr %v", c.desc, c.mr, mr) + } + } + } +} diff --git a/src/pkg/exp/draw/draw.go b/src/pkg/image/draw/draw.go index 1d0729d92..618fb4aa6 100644 --- a/src/pkg/exp/draw/draw.go +++ b/src/pkg/image/draw/draw.go @@ -2,18 +2,21 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package draw provides basic graphics and drawing primitives, +// Package draw provides image composition functions // in the style of the Plan 9 graphics library // (see http://plan9.bell-labs.com/magic/man2html/2/draw) // and the X Render extension. package draw -import "image" +import ( + "image" + "image/ycbcr" +) // m is the maximum color value returned by image.Color.RGBA. const m = 1<<16 - 1 -// A Porter-Duff compositing operator. +// Op is a Porter-Duff compositing operator. type Op int const ( @@ -36,27 +39,31 @@ func Draw(dst Image, r image.Rectangle, src image.Image, sp image.Point) { DrawMask(dst, r, src, sp, nil, image.ZP, Over) } -// DrawMask aligns r.Min in dst with sp in src and mp in mask and then replaces the rectangle r -// in dst with the result of a Porter-Duff composition. A nil mask is treated as opaque. -func DrawMask(dst Image, r image.Rectangle, src image.Image, sp image.Point, mask image.Image, mp image.Point, op Op) { - sb := src.Bounds() - dx, dy := sb.Max.X-sp.X, sb.Max.Y-sp.Y +// clip clips r against each image's bounds (after translating into the +// destination image's co-ordinate space) and shifts the points sp and mp by +// the same amount as the change in r.Min. +func clip(dst Image, r *image.Rectangle, src image.Image, sp *image.Point, mask image.Image, mp *image.Point) { + orig := r.Min + *r = r.Intersect(dst.Bounds()) + *r = r.Intersect(src.Bounds().Add(orig.Sub(*sp))) if mask != nil { - mb := mask.Bounds() - if dx > mb.Max.X-mp.X { - dx = mb.Max.X - mp.X - } - if dy > mb.Max.Y-mp.Y { - dy = mb.Max.Y - mp.Y - } + *r = r.Intersect(mask.Bounds().Add(orig.Sub(*mp))) } - if r.Dx() > dx { - r.Max.X = r.Min.X + dx - } - if r.Dy() > dy { - r.Max.Y = r.Min.Y + dy + dx := r.Min.X - orig.X + dy := r.Min.Y - orig.Y + if dx == 0 && dy == 0 { + return } - r = r.Intersect(dst.Bounds()) + (*sp).X += dx + (*sp).Y += dy + (*mp).X += dx + (*mp).Y += dy +} + +// DrawMask aligns r.Min in dst with sp in src and mp in mask and then replaces the rectangle r +// in dst with the result of a Porter-Duff composition. A nil mask is treated as opaque. +func DrawMask(dst Image, r image.Rectangle, src image.Image, sp image.Point, mask image.Image, mp image.Point, op Op) { + clip(dst, &r, src, &sp, mask, &mp) if r.Empty() { return } @@ -65,29 +72,42 @@ func DrawMask(dst Image, r image.Rectangle, src image.Image, sp image.Point, mas if dst0, ok := dst.(*image.RGBA); ok { if op == Over { if mask == nil { - if src0, ok := src.(*image.ColorImage); ok { + switch src0 := src.(type) { + case *image.ColorImage: drawFillOver(dst0, r, src0) return - } - if src0, ok := src.(*image.RGBA); ok { + case *image.RGBA: drawCopyOver(dst0, r, src0, sp) return + case *image.NRGBA: + drawNRGBAOver(dst0, r, src0, sp) + return + case *ycbcr.YCbCr: + drawYCbCr(dst0, r, src0, sp) + return } } else if mask0, ok := mask.(*image.Alpha); ok { - if src0, ok := src.(*image.ColorImage); ok { + switch src0 := src.(type) { + case *image.ColorImage: drawGlyphOver(dst0, r, src0, mask0, mp) return } } } else { if mask == nil { - if src0, ok := src.(*image.ColorImage); ok { + switch src0 := src.(type) { + case *image.ColorImage: drawFillSrc(dst0, r, src0) return - } - if src0, ok := src.(*image.RGBA); ok { + case *image.RGBA: drawCopySrc(dst0, r, src0, sp) return + case *image.NRGBA: + drawNRGBASrc(dst0, r, src0, sp) + return + case *ycbcr.YCbCr: + drawYCbCr(dst0, r, src0, sp) + return } } } @@ -224,6 +244,36 @@ func drawCopyOver(dst *image.RGBA, r image.Rectangle, src *image.RGBA, sp image. } } +func drawNRGBAOver(dst *image.RGBA, r image.Rectangle, src *image.NRGBA, sp image.Point) { + for y, sy := r.Min.Y, sp.Y; y != r.Max.Y; y, sy = y+1, sy+1 { + dpix := dst.Pix[y*dst.Stride : (y+1)*dst.Stride] + spix := src.Pix[sy*src.Stride : (sy+1)*src.Stride] + for x, sx := r.Min.X, sp.X; x != r.Max.X; x, sx = x+1, sx+1 { + // Convert from non-premultiplied color to pre-multiplied color. + // The order of operations here is to match the NRGBAColor.RGBA + // method in image/color.go. + snrgba := spix[sx] + sa := uint32(snrgba.A) + sr := uint32(snrgba.R) * 0x101 * sa / 0xff + sg := uint32(snrgba.G) * 0x101 * sa / 0xff + sb := uint32(snrgba.B) * 0x101 * sa / 0xff + sa *= 0x101 + + rgba := dpix[x] + dr := uint32(rgba.R) + dg := uint32(rgba.G) + db := uint32(rgba.B) + da := uint32(rgba.A) + a := (m - sa) * 0x101 + dr = (dr*a + sr*m) / m + dg = (dg*a + sg*m) / m + db = (db*a + sb*m) / m + da = (da*a + sa*m) / m + dpix[x] = image.RGBAColor{uint8(dr >> 8), uint8(dg >> 8), uint8(db >> 8), uint8(da >> 8)} + } + } +} + func drawGlyphOver(dst *image.RGBA, r image.Rectangle, src *image.ColorImage, mask *image.Alpha, mp image.Point) { x0, x1 := r.Min.X, r.Max.X y0, y1 := r.Min.Y, r.Max.Y @@ -311,6 +361,73 @@ func drawCopySrc(dst *image.RGBA, r image.Rectangle, src *image.RGBA, sp image.P } } +func drawNRGBASrc(dst *image.RGBA, r image.Rectangle, src *image.NRGBA, sp image.Point) { + for y, sy := r.Min.Y, sp.Y; y != r.Max.Y; y, sy = y+1, sy+1 { + dpix := dst.Pix[y*dst.Stride : (y+1)*dst.Stride] + spix := src.Pix[sy*src.Stride : (sy+1)*src.Stride] + for x, sx := r.Min.X, sp.X; x != r.Max.X; x, sx = x+1, sx+1 { + // Convert from non-premultiplied color to pre-multiplied color. + // The order of operations here is to match the NRGBAColor.RGBA + // method in image/color.go. + snrgba := spix[sx] + sa := uint32(snrgba.A) + sr := uint32(snrgba.R) * 0x101 * sa / 0xff + sg := uint32(snrgba.G) * 0x101 * sa / 0xff + sb := uint32(snrgba.B) * 0x101 * sa / 0xff + sa *= 0x101 + + dpix[x] = image.RGBAColor{uint8(sr >> 8), uint8(sg >> 8), uint8(sb >> 8), uint8(sa >> 8)} + } + } +} + +func drawYCbCr(dst *image.RGBA, r image.Rectangle, src *ycbcr.YCbCr, sp image.Point) { + // A YCbCr image is always fully opaque, and so if the mask is implicitly nil + // (i.e. fully opaque) then the op is effectively always Src. + var ( + yy, cb, cr uint8 + rr, gg, bb uint8 + ) + switch src.SubsampleRatio { + case ycbcr.SubsampleRatio422: + for y, sy := r.Min.Y, sp.Y; y != r.Max.Y; y, sy = y+1, sy+1 { + dpix := dst.Pix[y*dst.Stride : (y+1)*dst.Stride] + for x, sx := r.Min.X, sp.X; x != r.Max.X; x, sx = x+1, sx+1 { + i := sx / 2 + yy = src.Y[sy*src.YStride+sx] + cb = src.Cb[sy*src.CStride+i] + cr = src.Cr[sy*src.CStride+i] + rr, gg, bb = ycbcr.YCbCrToRGB(yy, cb, cr) + dpix[x] = image.RGBAColor{rr, gg, bb, 255} + } + } + case ycbcr.SubsampleRatio420: + for y, sy := r.Min.Y, sp.Y; y != r.Max.Y; y, sy = y+1, sy+1 { + dpix := dst.Pix[y*dst.Stride : (y+1)*dst.Stride] + for x, sx := r.Min.X, sp.X; x != r.Max.X; x, sx = x+1, sx+1 { + i, j := sx/2, sy/2 + yy = src.Y[sy*src.YStride+sx] + cb = src.Cb[j*src.CStride+i] + cr = src.Cr[j*src.CStride+i] + rr, gg, bb = ycbcr.YCbCrToRGB(yy, cb, cr) + dpix[x] = image.RGBAColor{rr, gg, bb, 255} + } + } + default: + // Default to 4:4:4 subsampling. + for y, sy := r.Min.Y, sp.Y; y != r.Max.Y; y, sy = y+1, sy+1 { + dpix := dst.Pix[y*dst.Stride : (y+1)*dst.Stride] + for x, sx := r.Min.X, sp.X; x != r.Max.X; x, sx = x+1, sx+1 { + yy = src.Y[sy*src.YStride+sx] + cb = src.Cb[sy*src.CStride+sx] + cr = src.Cr[sy*src.CStride+sx] + rr, gg, bb = ycbcr.YCbCrToRGB(yy, cb, cr) + dpix[x] = image.RGBAColor{rr, gg, bb, 255} + } + } + } +} + func drawRGBA(dst *image.RGBA, r image.Rectangle, src image.Image, sp image.Point, mask image.Image, mp image.Point, op Op) { x0, x1, dx := r.Min.X, r.Max.X, 1 y0, y1, dy := r.Min.Y, r.Max.Y, 1 diff --git a/src/pkg/exp/draw/draw_test.go b/src/pkg/image/draw/draw_test.go index 90c9e823d..37d630353 100644 --- a/src/pkg/exp/draw/draw_test.go +++ b/src/pkg/image/draw/draw_test.go @@ -6,6 +6,7 @@ package draw import ( "image" + "image/ycbcr" "testing" ) @@ -43,6 +44,34 @@ func vgradAlpha(alpha int) image.Image { return m } +func vgradGreenNRGBA(alpha int) image.Image { + m := image.NewNRGBA(16, 16) + for y := 0; y < 16; y++ { + for x := 0; x < 16; x++ { + m.Set(x, y, image.RGBAColor{0, uint8(y * 0x11), 0, uint8(alpha)}) + } + } + return m +} + +func vgradCr() image.Image { + m := &ycbcr.YCbCr{ + Y: make([]byte, 16*16), + Cb: make([]byte, 16*16), + Cr: make([]byte, 16*16), + YStride: 16, + CStride: 16, + SubsampleRatio: ycbcr.SubsampleRatio444, + Rect: image.Rect(0, 0, 16, 16), + } + for y := 0; y < 16; y++ { + for x := 0; x < 16; x++ { + m.Cr[y*m.CStride+x] = uint8(y * 0x11) + } + } + return m +} + func hgradRed(alpha int) Image { m := image.NewRGBA(16, 16) for y := 0; y < 16; y++ { @@ -95,6 +124,27 @@ var drawTests = []drawTest{ {"copyAlphaSrc", vgradGreen(90), fillAlpha(192), Src, image.RGBAColor{0, 36, 0, 68}}, {"copyNil", vgradGreen(90), nil, Over, image.RGBAColor{88, 48, 0, 255}}, {"copyNilSrc", vgradGreen(90), nil, Src, image.RGBAColor{0, 48, 0, 90}}, + // Uniform mask (100%, 75%, nil) and variable NRGBA source. + // At (x, y) == (8, 8): + // The destination pixel is {136, 0, 0, 255}. + // The source pixel is {0, 136, 0, 90} in NRGBA-space, which is {0, 48, 0, 90} in RGBA-space. + // The result pixel is different than in the "copy*" test cases because of rounding errors. + {"nrgba", vgradGreenNRGBA(90), fillAlpha(255), Over, image.RGBAColor{88, 46, 0, 255}}, + {"nrgbaSrc", vgradGreenNRGBA(90), fillAlpha(255), Src, image.RGBAColor{0, 46, 0, 90}}, + {"nrgbaAlpha", vgradGreenNRGBA(90), fillAlpha(192), Over, image.RGBAColor{100, 34, 0, 255}}, + {"nrgbaAlphaSrc", vgradGreenNRGBA(90), fillAlpha(192), Src, image.RGBAColor{0, 34, 0, 68}}, + {"nrgbaNil", vgradGreenNRGBA(90), nil, Over, image.RGBAColor{88, 46, 0, 255}}, + {"nrgbaNilSrc", vgradGreenNRGBA(90), nil, Src, image.RGBAColor{0, 46, 0, 90}}, + // Uniform mask (100%, 75%, nil) and variable YCbCr source. + // At (x, y) == (8, 8): + // The destination pixel is {136, 0, 0, 255}. + // The source pixel is {0, 0, 136} in YCbCr-space, which is {11, 38, 0, 255} in RGB-space. + {"ycbcr", vgradCr(), fillAlpha(255), Over, image.RGBAColor{11, 38, 0, 255}}, + {"ycbcrSrc", vgradCr(), fillAlpha(255), Src, image.RGBAColor{11, 38, 0, 255}}, + {"ycbcrAlpha", vgradCr(), fillAlpha(192), Over, image.RGBAColor{42, 28, 0, 255}}, + {"ycbcrAlphaSrc", vgradCr(), fillAlpha(192), Src, image.RGBAColor{8, 28, 0, 192}}, + {"ycbcrNil", vgradCr(), nil, Over, image.RGBAColor{11, 38, 0, 255}}, + {"ycbcrNilSrc", vgradCr(), nil, Src, image.RGBAColor{11, 38, 0, 255}}, // Variable mask and variable source. // At (x, y) == (8, 8): // The destination pixel is {136, 0, 0, 255}. @@ -213,8 +263,8 @@ func TestDrawOverlap(t *testing.T) { } } -// TestIssue836 verifies http://code.google.com/p/go/issues/detail?id=836. -func TestIssue836(t *testing.T) { +// TestNonZeroSrcPt checks drawing with a non-zero src point parameter. +func TestNonZeroSrcPt(t *testing.T) { a := image.NewRGBA(1, 1) b := image.NewRGBA(2, 2) b.Set(0, 0, image.RGBAColor{0, 0, 0, 5}) @@ -223,6 +273,6 @@ func TestIssue836(t *testing.T) { b.Set(1, 1, image.RGBAColor{5, 0, 0, 5}) Draw(a, image.Rect(0, 0, 1, 1), b, image.Pt(1, 1)) if !eq(image.RGBAColor{5, 0, 0, 5}, a.At(0, 0)) { - t.Errorf("Issue 836: want %v got %v", image.RGBAColor{5, 0, 0, 5}, a.At(0, 0)) + t.Errorf("non-zero src pt: want %v got %v", image.RGBAColor{5, 0, 0, 5}, a.At(0, 0)) } } diff --git a/src/pkg/image/geom.go b/src/pkg/image/geom.go index ccfe9cdb0..667aee625 100644 --- a/src/pkg/image/geom.go +++ b/src/pkg/image/geom.go @@ -38,6 +38,12 @@ func (p Point) Div(k int) Point { return Point{p.X / k, p.Y / k} } +// In returns whether p is in r. +func (p Point) In(r Rectangle) bool { + return r.Min.X <= p.X && p.X < r.Max.X && + r.Min.Y <= p.Y && p.Y < r.Max.Y +} + // Mod returns the point q in r such that p.X-q.X is a multiple of r's width // and p.Y-q.Y is a multiple of r's height. func (p Point) Mod(r Rectangle) Point { @@ -190,10 +196,15 @@ func (r Rectangle) Overlaps(s Rectangle) bool { r.Min.Y < s.Max.Y && s.Min.Y < r.Max.Y } -// Contains returns whether r contains p. -func (r Rectangle) Contains(p Point) bool { - return p.X >= r.Min.X && p.X < r.Max.X && - p.Y >= r.Min.Y && p.Y < r.Max.Y +// In returns whether every point in r is in s. +func (r Rectangle) In(s Rectangle) bool { + if r.Empty() { + return true + } + // Note that r.Max is an exclusive bound for r, so that r.In(s) + // does not require that r.Max.In(s). + return s.Min.X <= r.Min.X && r.Max.X <= s.Max.X && + s.Min.Y <= r.Min.Y && r.Max.Y <= s.Max.Y } // Canon returns the canonical version of r. The returned rectangle has minimum diff --git a/src/pkg/image/gif/Makefile b/src/pkg/image/gif/Makefile new file mode 100644 index 000000000..e89a71361 --- /dev/null +++ b/src/pkg/image/gif/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=image/gif +GOFILES=\ + reader.go\ + +include ../../../Make.pkg diff --git a/src/pkg/image/gif/reader.go b/src/pkg/image/gif/reader.go new file mode 100644 index 000000000..26c013b9a --- /dev/null +++ b/src/pkg/image/gif/reader.go @@ -0,0 +1,421 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gif implements a GIF image decoder. +// +// The GIF specification is at http://www.w3.org/Graphics/GIF/spec-gif89a.txt. +package gif + +import ( + "bufio" + "compress/lzw" + "fmt" + "image" + "io" + "os" +) + +// If the io.Reader does not also have ReadByte, then decode will introduce its own buffering. +type reader interface { + io.Reader + io.ByteReader +} + +// Masks etc. +const ( + // Fields. + fColorMapFollows = 1 << 7 + + // Image fields. + ifInterlace = 1 << 6 + + // Graphic control flags. + gcTransparentColorSet = 1 << 0 +) + +// Section indicators. +const ( + sExtension = 0x21 + sImageDescriptor = 0x2C + sTrailer = 0x3B +) + +// Extensions. +const ( + eText = 0x01 // Plain Text + eGraphicControl = 0xF9 // Graphic Control + eComment = 0xFE // Comment + eApplication = 0xFF // Application +) + +// decoder is the type used to decode a GIF file. +type decoder struct { + r reader + + // From header. + vers string + width int + height int + flags byte + headerFields byte + backgroundIndex byte + loopCount int + delayTime int + + // Unused from header. + aspect byte + + // From image descriptor. + imageFields byte + + // From graphics control. + transparentIndex byte + + // Computed. + pixelSize uint + globalColorMap image.PalettedColorModel + + // Used when decoding. + delay []int + image []*image.Paletted + tmp [1024]byte // must be at least 768 so we can read color map +} + +// blockReader parses the block structure of GIF image data, which +// comprises (n, (n bytes)) blocks, with 1 <= n <= 255. It is the +// reader given to the LZW decoder, which is thus immune to the +// blocking. After the LZW decoder completes, there will be a 0-byte +// block remaining (0, ()), but under normal execution blockReader +// doesn't consume it, so it is handled in decode. +type blockReader struct { + r reader + slice []byte + tmp [256]byte +} + +func (b *blockReader) Read(p []byte) (int, os.Error) { + if len(p) == 0 { + return 0, nil + } + if len(b.slice) == 0 { + blockLen, err := b.r.ReadByte() + if err != nil { + return 0, err + } + if blockLen == 0 { + return 0, os.EOF + } + b.slice = b.tmp[0:blockLen] + if _, err = io.ReadFull(b.r, b.slice); err != nil { + return 0, err + } + } + n := copy(p, b.slice) + b.slice = b.slice[n:] + return n, nil +} + +// decode reads a GIF image from r and stores the result in d. +func (d *decoder) decode(r io.Reader, configOnly bool) os.Error { + // Add buffering if r does not provide ReadByte. + if rr, ok := r.(reader); ok { + d.r = rr + } else { + d.r = bufio.NewReader(r) + } + + err := d.readHeaderAndScreenDescriptor() + if err != nil { + return err + } + if configOnly { + return nil + } + + if d.headerFields&fColorMapFollows != 0 { + if d.globalColorMap, err = d.readColorMap(); err != nil { + return err + } + } + +Loop: + for err == nil { + var c byte + c, err = d.r.ReadByte() + if err == os.EOF { + break + } + switch c { + case sExtension: + err = d.readExtension() + + case sImageDescriptor: + var m *image.Paletted + m, err = d.newImageFromDescriptor() + if err != nil { + break + } + if d.imageFields&fColorMapFollows != 0 { + m.Palette, err = d.readColorMap() + if err != nil { + break + } + // TODO: do we set transparency in this map too? That would be + // d.setTransparency(m.Palette) + } else { + m.Palette = d.globalColorMap + } + var litWidth uint8 + litWidth, err = d.r.ReadByte() + if err != nil { + return err + } + if litWidth < 2 || litWidth > 8 { + return fmt.Errorf("gif: pixel size in decode out of range: %d", litWidth) + } + // A wonderfully Go-like piece of magic. + lzwr := lzw.NewReader(&blockReader{r: d.r}, lzw.LSB, int(litWidth)) + if _, err = io.ReadFull(lzwr, m.Pix); err != nil { + break + } + + // There should be a "0" block remaining; drain that. + c, err = d.r.ReadByte() + if err != nil { + return err + } + if c != 0 { + return os.ErrorString("gif: extra data after image") + } + + // Undo the interlacing if necessary. + d.uninterlace(m) + + d.image = append(d.image, m) + d.delay = append(d.delay, d.delayTime) + d.delayTime = 0 // TODO: is this correct, or should we hold on to the value? + + case sTrailer: + break Loop + + default: + err = fmt.Errorf("gif: unknown block type: 0x%.2x", c) + } + } + if err != nil { + return err + } + if len(d.image) == 0 { + return io.ErrUnexpectedEOF + } + return nil +} + +func (d *decoder) readHeaderAndScreenDescriptor() os.Error { + _, err := io.ReadFull(d.r, d.tmp[0:13]) + if err != nil { + return err + } + d.vers = string(d.tmp[0:6]) + if d.vers != "GIF87a" && d.vers != "GIF89a" { + return fmt.Errorf("gif: can't recognize format %s", d.vers) + } + d.width = int(d.tmp[6]) + int(d.tmp[7])<<8 + d.height = int(d.tmp[8]) + int(d.tmp[9])<<8 + d.headerFields = d.tmp[10] + d.backgroundIndex = d.tmp[11] + d.aspect = d.tmp[12] + d.loopCount = -1 + d.pixelSize = uint(d.headerFields&7) + 1 + return nil +} + +func (d *decoder) readColorMap() (image.PalettedColorModel, os.Error) { + if d.pixelSize > 8 { + return nil, fmt.Errorf("gif: can't handle %d bits per pixel", d.pixelSize) + } + numColors := 1 << d.pixelSize + numValues := 3 * numColors + _, err := io.ReadFull(d.r, d.tmp[0:numValues]) + if err != nil { + return nil, fmt.Errorf("gif: short read on color map: %s", err) + } + colorMap := make(image.PalettedColorModel, numColors) + j := 0 + for i := range colorMap { + colorMap[i] = image.RGBAColor{d.tmp[j+0], d.tmp[j+1], d.tmp[j+2], 0xFF} + j += 3 + } + return colorMap, nil +} + +func (d *decoder) readExtension() os.Error { + extension, err := d.r.ReadByte() + if err != nil { + return err + } + size := 0 + switch extension { + case eText: + size = 13 + case eGraphicControl: + return d.readGraphicControl() + case eComment: + // nothing to do but read the data. + case eApplication: + b, err := d.r.ReadByte() + if err != nil { + return err + } + // The spec requires size be 11, but Adobe sometimes uses 10. + size = int(b) + default: + return fmt.Errorf("gif: unknown extension 0x%.2x", extension) + } + if size > 0 { + if _, err := io.ReadFull(d.r, d.tmp[0:size]); err != nil { + return err + } + } + + // Application Extension with "NETSCAPE2.0" as string and 1 in data means + // this extension defines a loop count. + if extension == eApplication && string(d.tmp[:size]) == "NETSCAPE2.0" { + n, err := d.readBlock() + if n == 0 || err != nil { + return err + } + if n == 3 && d.tmp[0] == 1 { + d.loopCount = int(d.tmp[1]) | int(d.tmp[2])<<8 + } + } + for { + n, err := d.readBlock() + if n == 0 || err != nil { + return err + } + } + panic("unreachable") +} + +func (d *decoder) readGraphicControl() os.Error { + if _, err := io.ReadFull(d.r, d.tmp[0:6]); err != nil { + return fmt.Errorf("gif: can't read graphic control: %s", err) + } + d.flags = d.tmp[1] + d.delayTime = int(d.tmp[2]) | int(d.tmp[3])<<8 + if d.flags&gcTransparentColorSet != 0 { + d.transparentIndex = d.tmp[4] + d.setTransparency(d.globalColorMap) + } + return nil +} + +func (d *decoder) setTransparency(colorMap image.PalettedColorModel) { + if int(d.transparentIndex) < len(colorMap) { + colorMap[d.transparentIndex] = image.RGBAColor{} + } +} + +func (d *decoder) newImageFromDescriptor() (*image.Paletted, os.Error) { + if _, err := io.ReadFull(d.r, d.tmp[0:9]); err != nil { + return nil, fmt.Errorf("gif: can't read image descriptor: %s", err) + } + // TODO: This code (throughout) ignores the top and left values, + // and assumes (in interlacing, for example) that the images' + // widths and heights are all the same. + _ = int(d.tmp[0]) + int(d.tmp[1])<<8 // TODO: honor left value + _ = int(d.tmp[2]) + int(d.tmp[3])<<8 // TODO: honor top value + width := int(d.tmp[4]) + int(d.tmp[5])<<8 + height := int(d.tmp[6]) + int(d.tmp[7])<<8 + d.imageFields = d.tmp[8] + return image.NewPaletted(width, height, nil), nil +} + +func (d *decoder) readBlock() (int, os.Error) { + n, err := d.r.ReadByte() + if n == 0 || err != nil { + return 0, err + } + return io.ReadFull(d.r, d.tmp[0:n]) +} + +// interlaceScan defines the ordering for a pass of the interlace algorithm. +type interlaceScan struct { + skip, start int +} + +// interlacing represents the set of scans in an interlaced GIF image. +var interlacing = []interlaceScan{ + {8, 0}, // Group 1 : Every 8th. row, starting with row 0. + {8, 4}, // Group 2 : Every 8th. row, starting with row 4. + {4, 2}, // Group 3 : Every 4th. row, starting with row 2. + {2, 1}, // Group 4 : Every 2nd. row, starting with row 1. +} + +func (d *decoder) uninterlace(m *image.Paletted) { + if d.imageFields&ifInterlace == 0 { + return + } + var nPix []uint8 + dx := d.width + dy := d.height + nPix = make([]uint8, dx*dy) + offset := 0 // steps through the input by sequential scan lines. + for _, pass := range interlacing { + nOffset := pass.start * dx // steps through the output as defined by pass. + for y := pass.start; y < dy; y += pass.skip { + copy(nPix[nOffset:nOffset+dx], m.Pix[offset:offset+dx]) + offset += dx + nOffset += dx * pass.skip + } + } + m.Pix = nPix +} + +// Decode reads a GIF image from r and returns the first embedded +// image as an image.Image. +func Decode(r io.Reader) (image.Image, os.Error) { + var d decoder + if err := d.decode(r, false); err != nil { + return nil, err + } + return d.image[0], nil +} + +// GIF represents the possibly multiple images stored in a GIF file. +type GIF struct { + Image []*image.Paletted // The successive images. + Delay []int // The successive delay times, one per frame, in 100ths of a second. + LoopCount int // The loop count. +} + +// DecodeAll reads a GIF image from r and returns the sequential frames +// and timing information. +func DecodeAll(r io.Reader) (*GIF, os.Error) { + var d decoder + if err := d.decode(r, false); err != nil { + return nil, err + } + gif := &GIF{ + Image: d.image, + LoopCount: d.loopCount, + Delay: d.delay, + } + return gif, nil +} + +// DecodeConfig returns the global color model and dimensions of a GIF image +// without decoding the entire image. +func DecodeConfig(r io.Reader) (image.Config, os.Error) { + var d decoder + if err := d.decode(r, true); err != nil { + return image.Config{}, err + } + return image.Config{d.globalColorMap, d.width, d.height}, nil +} + +func init() { + image.RegisterFormat("gif", "GIF8?a", Decode, DecodeConfig) +} diff --git a/src/pkg/image/image.go b/src/pkg/image/image.go index 222d21ade..f4c38d28a 100644 --- a/src/pkg/image/image.go +++ b/src/pkg/image/image.go @@ -38,26 +38,36 @@ func (p *RGBA) ColorModel() ColorModel { return RGBAColorModel } func (p *RGBA) Bounds() Rectangle { return p.Rect } func (p *RGBA) At(x, y int) Color { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return RGBAColor{} } return p.Pix[y*p.Stride+x] } func (p *RGBA) Set(x, y int, c Color) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = toRGBAColor(c).(RGBAColor) } func (p *RGBA) SetRGBA(x, y int, c RGBAColor) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = c } +// SubImage returns an image representing the portion of the image p visible +// through r. The returned value shares pixels with the original image. +func (p *RGBA) SubImage(r Rectangle) Image { + return &RGBA{ + Pix: p.Pix, + Stride: p.Stride, + Rect: p.Rect.Intersect(r), + } +} + // Opaque scans the entire image and returns whether or not it is fully opaque. func (p *RGBA) Opaque() bool { if p.Rect.Empty() { @@ -97,26 +107,36 @@ func (p *RGBA64) ColorModel() ColorModel { return RGBA64ColorModel } func (p *RGBA64) Bounds() Rectangle { return p.Rect } func (p *RGBA64) At(x, y int) Color { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return RGBA64Color{} } return p.Pix[y*p.Stride+x] } func (p *RGBA64) Set(x, y int, c Color) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = toRGBA64Color(c).(RGBA64Color) } func (p *RGBA64) SetRGBA64(x, y int, c RGBA64Color) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = c } +// SubImage returns an image representing the portion of the image p visible +// through r. The returned value shares pixels with the original image. +func (p *RGBA64) SubImage(r Rectangle) Image { + return &RGBA64{ + Pix: p.Pix, + Stride: p.Stride, + Rect: p.Rect.Intersect(r), + } +} + // Opaque scans the entire image and returns whether or not it is fully opaque. func (p *RGBA64) Opaque() bool { if p.Rect.Empty() { @@ -156,26 +176,36 @@ func (p *NRGBA) ColorModel() ColorModel { return NRGBAColorModel } func (p *NRGBA) Bounds() Rectangle { return p.Rect } func (p *NRGBA) At(x, y int) Color { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return NRGBAColor{} } return p.Pix[y*p.Stride+x] } func (p *NRGBA) Set(x, y int, c Color) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = toNRGBAColor(c).(NRGBAColor) } func (p *NRGBA) SetNRGBA(x, y int, c NRGBAColor) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = c } +// SubImage returns an image representing the portion of the image p visible +// through r. The returned value shares pixels with the original image. +func (p *NRGBA) SubImage(r Rectangle) Image { + return &NRGBA{ + Pix: p.Pix, + Stride: p.Stride, + Rect: p.Rect.Intersect(r), + } +} + // Opaque scans the entire image and returns whether or not it is fully opaque. func (p *NRGBA) Opaque() bool { if p.Rect.Empty() { @@ -215,26 +245,36 @@ func (p *NRGBA64) ColorModel() ColorModel { return NRGBA64ColorModel } func (p *NRGBA64) Bounds() Rectangle { return p.Rect } func (p *NRGBA64) At(x, y int) Color { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return NRGBA64Color{} } return p.Pix[y*p.Stride+x] } func (p *NRGBA64) Set(x, y int, c Color) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = toNRGBA64Color(c).(NRGBA64Color) } func (p *NRGBA64) SetNRGBA64(x, y int, c NRGBA64Color) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = c } +// SubImage returns an image representing the portion of the image p visible +// through r. The returned value shares pixels with the original image. +func (p *NRGBA64) SubImage(r Rectangle) Image { + return &NRGBA64{ + Pix: p.Pix, + Stride: p.Stride, + Rect: p.Rect.Intersect(r), + } +} + // Opaque scans the entire image and returns whether or not it is fully opaque. func (p *NRGBA64) Opaque() bool { if p.Rect.Empty() { @@ -274,26 +314,36 @@ func (p *Alpha) ColorModel() ColorModel { return AlphaColorModel } func (p *Alpha) Bounds() Rectangle { return p.Rect } func (p *Alpha) At(x, y int) Color { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return AlphaColor{} } return p.Pix[y*p.Stride+x] } -func (p *Alpha) Set(x, y int, c AlphaColor) { - if !p.Rect.Contains(Point{x, y}) { +func (p *Alpha) Set(x, y int, c Color) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = toAlphaColor(c).(AlphaColor) } func (p *Alpha) SetAlpha(x, y int, c AlphaColor) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = c } +// SubImage returns an image representing the portion of the image p visible +// through r. The returned value shares pixels with the original image. +func (p *Alpha) SubImage(r Rectangle) Image { + return &Alpha{ + Pix: p.Pix, + Stride: p.Stride, + Rect: p.Rect.Intersect(r), + } +} + // Opaque scans the entire image and returns whether or not it is fully opaque. func (p *Alpha) Opaque() bool { if p.Rect.Empty() { @@ -333,26 +383,36 @@ func (p *Alpha16) ColorModel() ColorModel { return Alpha16ColorModel } func (p *Alpha16) Bounds() Rectangle { return p.Rect } func (p *Alpha16) At(x, y int) Color { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return Alpha16Color{} } return p.Pix[y*p.Stride+x] } func (p *Alpha16) Set(x, y int, c Color) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = toAlpha16Color(c).(Alpha16Color) } func (p *Alpha16) SetAlpha16(x, y int, c Alpha16Color) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = c } +// SubImage returns an image representing the portion of the image p visible +// through r. The returned value shares pixels with the original image. +func (p *Alpha16) SubImage(r Rectangle) Image { + return &Alpha16{ + Pix: p.Pix, + Stride: p.Stride, + Rect: p.Rect.Intersect(r), + } +} + // Opaque scans the entire image and returns whether or not it is fully opaque. func (p *Alpha16) Opaque() bool { if p.Rect.Empty() { @@ -392,26 +452,36 @@ func (p *Gray) ColorModel() ColorModel { return GrayColorModel } func (p *Gray) Bounds() Rectangle { return p.Rect } func (p *Gray) At(x, y int) Color { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return GrayColor{} } return p.Pix[y*p.Stride+x] } func (p *Gray) Set(x, y int, c Color) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = toGrayColor(c).(GrayColor) } func (p *Gray) SetGray(x, y int, c GrayColor) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = c } +// SubImage returns an image representing the portion of the image p visible +// through r. The returned value shares pixels with the original image. +func (p *Gray) SubImage(r Rectangle) Image { + return &Gray{ + Pix: p.Pix, + Stride: p.Stride, + Rect: p.Rect.Intersect(r), + } +} + // Opaque scans the entire image and returns whether or not it is fully opaque. func (p *Gray) Opaque() bool { return true @@ -437,26 +507,36 @@ func (p *Gray16) ColorModel() ColorModel { return Gray16ColorModel } func (p *Gray16) Bounds() Rectangle { return p.Rect } func (p *Gray16) At(x, y int) Color { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return Gray16Color{} } return p.Pix[y*p.Stride+x] } func (p *Gray16) Set(x, y int, c Color) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = toGray16Color(c).(Gray16Color) } func (p *Gray16) SetGray16(x, y int, c Gray16Color) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = c } +// SubImage returns an image representing the portion of the image p visible +// through r. The returned value shares pixels with the original image. +func (p *Gray16) SubImage(r Rectangle) Image { + return &Gray16{ + Pix: p.Pix, + Stride: p.Stride, + Rect: p.Rect.Intersect(r), + } +} + // Opaque scans the entire image and returns whether or not it is fully opaque. func (p *Gray16) Opaque() bool { return true @@ -483,14 +563,19 @@ func (p PalettedColorModel) Convert(c Color) Color { if len(p) == 0 { return nil } + return p[p.Index(c)] +} + +// Index returns the index of the palette color closest to c in Euclidean +// R,G,B space. +func (p PalettedColorModel) Index(c Color) int { cr, cg, cb, _ := c.RGBA() // Shift by 1 bit to avoid potential uint32 overflow in sum-squared-difference. cr >>= 1 cg >>= 1 cb >>= 1 - result := Color(nil) - bestSSD := uint32(1<<32 - 1) - for _, v := range p { + ret, bestSSD := 0, uint32(1<<32-1) + for i, v := range p { vr, vg, vb, _ := v.RGBA() vr >>= 1 vg >>= 1 @@ -498,11 +583,10 @@ func (p PalettedColorModel) Convert(c Color) Color { dr, dg, db := diff(cr, vr), diff(cg, vg), diff(cb, vb) ssd := (dr * dr) + (dg * dg) + (db * db) if ssd < bestSSD { - bestSSD = ssd - result = v + ret, bestSSD = i, ssd } } - return result + return ret } // A Paletted is an in-memory image backed by a 2-D slice of uint8 values and a PalettedColorModel. @@ -524,26 +608,44 @@ func (p *Paletted) At(x, y int) Color { if len(p.Palette) == 0 { return nil } - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return p.Palette[0] } return p.Palette[p.Pix[y*p.Stride+x]] } +func (p *Paletted) Set(x, y int, c Color) { + if !(Point{x, y}.In(p.Rect)) { + return + } + p.Pix[y*p.Stride+x] = uint8(p.Palette.Index(c)) +} + func (p *Paletted) ColorIndexAt(x, y int) uint8 { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return 0 } return p.Pix[y*p.Stride+x] } func (p *Paletted) SetColorIndex(x, y int, index uint8) { - if !p.Rect.Contains(Point{x, y}) { + if !(Point{x, y}.In(p.Rect)) { return } p.Pix[y*p.Stride+x] = index } +// SubImage returns an image representing the portion of the image p visible +// through r. The returned value shares pixels with the original image. +func (p *Paletted) SubImage(r Rectangle) Image { + return &Paletted{ + Pix: p.Pix, + Stride: p.Stride, + Rect: p.Rect.Intersect(r), + Palette: p.Palette, + } +} + // Opaque scans the entire image and returns whether or not it is fully opaque. func (p *Paletted) Opaque() bool { for _, c := range p.Palette { diff --git a/src/pkg/image/image_test.go b/src/pkg/image/image_test.go new file mode 100644 index 000000000..17e314795 --- /dev/null +++ b/src/pkg/image/image_test.go @@ -0,0 +1,71 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package image + +import ( + "testing" +) + +func cmp(t *testing.T, cm ColorModel, c0, c1 Color) bool { + r0, g0, b0, a0 := cm.Convert(c0).RGBA() + r1, g1, b1, a1 := cm.Convert(c1).RGBA() + return r0 == r1 && g0 == g1 && b0 == b1 && a0 == a1 +} + +func TestImage(t *testing.T) { + type buffered interface { + Image + Set(int, int, Color) + SubImage(Rectangle) Image + } + testImage := []Image{ + NewRGBA(10, 10), + NewRGBA64(10, 10), + NewNRGBA(10, 10), + NewNRGBA64(10, 10), + NewAlpha(10, 10), + NewAlpha16(10, 10), + NewGray(10, 10), + NewGray16(10, 10), + NewPaletted(10, 10, PalettedColorModel{ + Transparent, + Opaque, + }), + } + for _, m := range testImage { + b := m.(buffered) + if !Rect(0, 0, 10, 10).Eq(b.Bounds()) { + t.Errorf("%T: want bounds %v, got %v", b, Rect(0, 0, 10, 10), b.Bounds()) + continue + } + if !cmp(t, b.ColorModel(), Transparent, b.At(6, 3)) { + t.Errorf("%T: at (6, 3), want a zero color, got %v", b, b.At(6, 3)) + continue + } + b.Set(6, 3, Opaque) + if !cmp(t, b.ColorModel(), Opaque, b.At(6, 3)) { + t.Errorf("%T: at (6, 3), want a non-zero color, got %v", b, b.At(6, 3)) + continue + } + b = b.SubImage(Rect(3, 2, 9, 8)).(buffered) + if !Rect(3, 2, 9, 8).Eq(b.Bounds()) { + t.Errorf("%T: sub-image want bounds %v, got %v", b, Rect(3, 2, 9, 8), b.Bounds()) + continue + } + if !cmp(t, b.ColorModel(), Opaque, b.At(6, 3)) { + t.Errorf("%T: sub-image at (6, 3), want a non-zero color, got %v", b, b.At(6, 3)) + continue + } + if !cmp(t, b.ColorModel(), Transparent, b.At(3, 3)) { + t.Errorf("%T: sub-image at (3, 3), want a zero color, got %v", b, b.At(3, 3)) + continue + } + b.Set(3, 3, Opaque) + if !cmp(t, b.ColorModel(), Opaque, b.At(3, 3)) { + t.Errorf("%T: sub-image at (3, 3), want a non-zero color, got %v", b, b.At(3, 3)) + continue + } + } +} diff --git a/src/pkg/image/jpeg/idct.go b/src/pkg/image/jpeg/idct.go index e5a2f40f5..b387dfdff 100644 --- a/src/pkg/image/jpeg/idct.go +++ b/src/pkg/image/jpeg/idct.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +package jpeg + // This is a Go translation of idct.c from // // http://standards.iso.org/ittf/PubliclyAvailableStandards/ISO_IEC_13818-4_2004_Conformance_Testing/Video/verifier/mpeg2decode_960109.tar.gz @@ -35,8 +37,6 @@ * */ -package jpeg - const ( w1 = 2841 // 2048*sqrt(2)*cos(1*pi/16) w2 = 2676 // 2048*sqrt(2)*cos(2*pi/16) @@ -55,41 +55,45 @@ const ( r2 = 181 // 256/sqrt(2) ) -// 2-D Inverse Discrete Cosine Transformation, followed by a +128 level shift. +// idct performs a 2-D Inverse Discrete Cosine Transformation, followed by a +// +128 level shift and a clip to [0, 255], writing the results to dst. +// stride is the number of elements between successive rows of dst. // -// The input coefficients should already have been multiplied by the appropriate quantization table. -// We use fixed-point computation, with the number of bits for the fractional component varying over the -// intermediate stages. The final values are expected to range within [0, 255], after a +128 level shift. +// The input coefficients should already have been multiplied by the +// appropriate quantization table. We use fixed-point computation, with the +// number of bits for the fractional component varying over the intermediate +// stages. // -// For more on the actual algorithm, see Z. Wang, "Fast algorithms for the discrete W transform and -// for the discrete Fourier transform", IEEE Trans. on ASSP, Vol. ASSP- 32, pp. 803-816, Aug. 1984. -func idct(b *block) { +// For more on the actual algorithm, see Z. Wang, "Fast algorithms for the +// discrete W transform and for the discrete Fourier transform", IEEE Trans. on +// ASSP, Vol. ASSP- 32, pp. 803-816, Aug. 1984. +func idct(dst []byte, stride int, src *block) { // Horizontal 1-D IDCT. for y := 0; y < 8; y++ { // If all the AC components are zero, then the IDCT is trivial. - if b[y*8+1] == 0 && b[y*8+2] == 0 && b[y*8+3] == 0 && - b[y*8+4] == 0 && b[y*8+5] == 0 && b[y*8+6] == 0 && b[y*8+7] == 0 { - dc := b[y*8+0] << 3 - b[y*8+0] = dc - b[y*8+1] = dc - b[y*8+2] = dc - b[y*8+3] = dc - b[y*8+4] = dc - b[y*8+5] = dc - b[y*8+6] = dc - b[y*8+7] = dc + if src[y*8+1] == 0 && src[y*8+2] == 0 && src[y*8+3] == 0 && + src[y*8+4] == 0 && src[y*8+5] == 0 && src[y*8+6] == 0 && src[y*8+7] == 0 { + dc := src[y*8+0] << 3 + src[y*8+0] = dc + src[y*8+1] = dc + src[y*8+2] = dc + src[y*8+3] = dc + src[y*8+4] = dc + src[y*8+5] = dc + src[y*8+6] = dc + src[y*8+7] = dc continue } // Prescale. - x0 := (b[y*8+0] << 11) + 128 - x1 := b[y*8+4] << 11 - x2 := b[y*8+6] - x3 := b[y*8+2] - x4 := b[y*8+1] - x5 := b[y*8+7] - x6 := b[y*8+5] - x7 := b[y*8+3] + x0 := (src[y*8+0] << 11) + 128 + x1 := src[y*8+4] << 11 + x2 := src[y*8+6] + x3 := src[y*8+2] + x4 := src[y*8+1] + x5 := src[y*8+7] + x6 := src[y*8+5] + x7 := src[y*8+3] // Stage 1. x8 := w7 * (x4 + x5) @@ -119,14 +123,14 @@ func idct(b *block) { x4 = (r2*(x4-x5) + 128) >> 8 // Stage 4. - b[8*y+0] = (x7 + x1) >> 8 - b[8*y+1] = (x3 + x2) >> 8 - b[8*y+2] = (x0 + x4) >> 8 - b[8*y+3] = (x8 + x6) >> 8 - b[8*y+4] = (x8 - x6) >> 8 - b[8*y+5] = (x0 - x4) >> 8 - b[8*y+6] = (x3 - x2) >> 8 - b[8*y+7] = (x7 - x1) >> 8 + src[8*y+0] = (x7 + x1) >> 8 + src[8*y+1] = (x3 + x2) >> 8 + src[8*y+2] = (x0 + x4) >> 8 + src[8*y+3] = (x8 + x6) >> 8 + src[8*y+4] = (x8 - x6) >> 8 + src[8*y+5] = (x0 - x4) >> 8 + src[8*y+6] = (x3 - x2) >> 8 + src[8*y+7] = (x7 - x1) >> 8 } // Vertical 1-D IDCT. @@ -136,14 +140,14 @@ func idct(b *block) { // we do not bother to check for the all-zero case. // Prescale. - y0 := (b[8*0+x] << 8) + 8192 - y1 := b[8*4+x] << 8 - y2 := b[8*6+x] - y3 := b[8*2+x] - y4 := b[8*1+x] - y5 := b[8*7+x] - y6 := b[8*5+x] - y7 := b[8*3+x] + y0 := (src[8*0+x] << 8) + 8192 + y1 := src[8*4+x] << 8 + y2 := src[8*6+x] + y3 := src[8*2+x] + y4 := src[8*1+x] + y5 := src[8*7+x] + y6 := src[8*5+x] + y7 := src[8*3+x] // Stage 1. y8 := w7*(y4+y5) + 4 @@ -173,18 +177,28 @@ func idct(b *block) { y4 = (r2*(y4-y5) + 128) >> 8 // Stage 4. - b[8*0+x] = (y7 + y1) >> 14 - b[8*1+x] = (y3 + y2) >> 14 - b[8*2+x] = (y0 + y4) >> 14 - b[8*3+x] = (y8 + y6) >> 14 - b[8*4+x] = (y8 - y6) >> 14 - b[8*5+x] = (y0 - y4) >> 14 - b[8*6+x] = (y3 - y2) >> 14 - b[8*7+x] = (y7 - y1) >> 14 + src[8*0+x] = (y7 + y1) >> 14 + src[8*1+x] = (y3 + y2) >> 14 + src[8*2+x] = (y0 + y4) >> 14 + src[8*3+x] = (y8 + y6) >> 14 + src[8*4+x] = (y8 - y6) >> 14 + src[8*5+x] = (y0 - y4) >> 14 + src[8*6+x] = (y3 - y2) >> 14 + src[8*7+x] = (y7 - y1) >> 14 } - // Level shift. - for i := range *b { - b[i] += 128 + // Level shift by +128, clip to [0, 255], and write to dst. + for y := 0; y < 8; y++ { + for x := 0; x < 8; x++ { + c := src[y*8+x] + if c < -128 { + c = 0 + } else if c > 127 { + c = 255 + } else { + c += 128 + } + dst[y*stride+x] = uint8(c) + } } } diff --git a/src/pkg/image/jpeg/reader.go b/src/pkg/image/jpeg/reader.go index 21a6fff96..ef8383a35 100644 --- a/src/pkg/image/jpeg/reader.go +++ b/src/pkg/image/jpeg/reader.go @@ -41,16 +41,22 @@ type block [blockSize]int const ( blockSize = 64 // A DCT block is 8x8. - dcTableClass = 0 - acTableClass = 1 - maxTc = 1 - maxTh = 3 - maxTq = 3 - - // We only support 4:4:4, 4:2:2 and 4:2:0 downsampling, and assume that the components are Y, Cb, Cr. - nComponent = 3 - maxH = 2 - maxV = 2 + dcTable = 0 + acTable = 1 + maxTc = 1 + maxTh = 3 + maxTq = 3 + + // A grayscale JPEG image has only a Y component. + nGrayComponent = 1 + // A color JPEG image has Y, Cb and Cr components. + nColorComponent = 3 + + // We only support 4:4:4, 4:2:2 and 4:2:0 downsampling, and therefore the + // number of luma samples per chroma sample is at most 2 in the horizontal + // and 2 in the vertical direction. + maxH = 2 + maxV = 2 ) const ( @@ -90,13 +96,14 @@ type Reader interface { type decoder struct { r Reader width, height int - img *ycbcr.YCbCr + img1 *image.Gray + img3 *ycbcr.YCbCr ri int // Restart Interval. - comps [nComponent]component + nComp int + comp [nColorComponent]component huff [maxTc + 1][maxTh + 1]huffman quant [maxTq + 1]block b bits - blocks [nComponent][maxH * maxV]block tmp [1024]byte } @@ -118,10 +125,15 @@ func (d *decoder) ignore(n int) os.Error { // Specified in section B.2.2. func (d *decoder) processSOF(n int) os.Error { - if n != 6+3*nComponent { + switch n { + case 6 + 3*nGrayComponent: + d.nComp = nGrayComponent + case 6 + 3*nColorComponent: + d.nComp = nColorComponent + default: return UnsupportedError("SOF has wrong length") } - _, err := io.ReadFull(d.r, d.tmp[0:6+3*nComponent]) + _, err := io.ReadFull(d.r, d.tmp[:n]) if err != nil { return err } @@ -131,26 +143,28 @@ func (d *decoder) processSOF(n int) os.Error { } d.height = int(d.tmp[1])<<8 + int(d.tmp[2]) d.width = int(d.tmp[3])<<8 + int(d.tmp[4]) - if d.tmp[5] != nComponent { + if int(d.tmp[5]) != d.nComp { return UnsupportedError("SOF has wrong number of image components") } - for i := 0; i < nComponent; i++ { + for i := 0; i < d.nComp; i++ { hv := d.tmp[7+3*i] - d.comps[i].h = int(hv >> 4) - d.comps[i].v = int(hv & 0x0f) - d.comps[i].c = d.tmp[6+3*i] - d.comps[i].tq = d.tmp[8+3*i] - // We only support YCbCr images, and 4:4:4, 4:2:2 or 4:2:0 chroma downsampling ratios. This implies that - // the (h, v) values for the Y component are either (1, 1), (2, 1) or (2, 2), and the - // (h, v) values for the Cr and Cb components must be (1, 1). + d.comp[i].h = int(hv >> 4) + d.comp[i].v = int(hv & 0x0f) + d.comp[i].c = d.tmp[6+3*i] + d.comp[i].tq = d.tmp[8+3*i] + if d.nComp == nGrayComponent { + continue + } + // For color images, we only support 4:4:4, 4:2:2 or 4:2:0 chroma + // downsampling ratios. This implies that the (h, v) values for the Y + // component are either (1, 1), (2, 1) or (2, 2), and the (h, v) + // values for the Cr and Cb components must be (1, 1). if i == 0 { if hv != 0x11 && hv != 0x21 && hv != 0x22 { return UnsupportedError("luma downsample ratio") } - } else { - if hv != 0x11 { - return UnsupportedError("chroma downsample ratio") - } + } else if hv != 0x11 { + return UnsupportedError("chroma downsample ratio") } } return nil @@ -182,110 +196,88 @@ func (d *decoder) processDQT(n int) os.Error { return nil } -// Clip x to the range [0, 255] inclusive. -func clip(x int) uint8 { - if x < 0 { - return 0 +// makeImg allocates and initializes the destination image. +func (d *decoder) makeImg(h0, v0, mxx, myy int) { + if d.nComp == nGrayComponent { + d.img1 = image.NewGray(8*mxx, 8*myy) + d.img1.Rect = image.Rect(0, 0, d.width, d.height) + return } - if x > 255 { - return 255 + var subsampleRatio ycbcr.SubsampleRatio + n := h0 * v0 + switch n { + case 1: + subsampleRatio = ycbcr.SubsampleRatio444 + case 2: + subsampleRatio = ycbcr.SubsampleRatio422 + case 4: + subsampleRatio = ycbcr.SubsampleRatio420 + default: + panic("unreachable") } - return uint8(x) -} - -// Store the MCU to the image. -func (d *decoder) storeMCU(mx, my int) { - h0, v0 := d.comps[0].h, d.comps[0].v - // Store the luma blocks. - for v := 0; v < v0; v++ { - for h := 0; h < h0; h++ { - p := 8 * ((v0*my+v)*d.img.YStride + (h0*mx + h)) - for y := 0; y < 8; y++ { - for x := 0; x < 8; x++ { - d.img.Y[p] = clip(d.blocks[0][h0*v+h][8*y+x]) - p++ - } - p += d.img.YStride - 8 - } - } - } - // Store the chroma blocks. - p := 8 * (my*d.img.CStride + mx) - for y := 0; y < 8; y++ { - for x := 0; x < 8; x++ { - d.img.Cb[p] = clip(d.blocks[1][0][8*y+x]) - d.img.Cr[p] = clip(d.blocks[2][0][8*y+x]) - p++ - } - p += d.img.CStride - 8 + b := make([]byte, mxx*myy*(1*8*8*n+2*8*8)) + d.img3 = &ycbcr.YCbCr{ + Y: b[mxx*myy*(0*8*8*n+0*8*8) : mxx*myy*(1*8*8*n+0*8*8)], + Cb: b[mxx*myy*(1*8*8*n+0*8*8) : mxx*myy*(1*8*8*n+1*8*8)], + Cr: b[mxx*myy*(1*8*8*n+1*8*8) : mxx*myy*(1*8*8*n+2*8*8)], + SubsampleRatio: subsampleRatio, + YStride: mxx * 8 * h0, + CStride: mxx * 8, + Rect: image.Rect(0, 0, d.width, d.height), } } // Specified in section B.2.3. func (d *decoder) processSOS(n int) os.Error { - if n != 4+2*nComponent { + if d.nComp == 0 { + return FormatError("missing SOF marker") + } + if n != 4+2*d.nComp { return UnsupportedError("SOS has wrong length") } - _, err := io.ReadFull(d.r, d.tmp[0:4+2*nComponent]) + _, err := io.ReadFull(d.r, d.tmp[0:4+2*d.nComp]) if err != nil { return err } - if d.tmp[0] != nComponent { + if int(d.tmp[0]) != d.nComp { return UnsupportedError("SOS has wrong number of image components") } - var scanComps [nComponent]struct { + var scan [nColorComponent]struct { td uint8 // DC table selector. ta uint8 // AC table selector. } - for i := 0; i < nComponent; i++ { + for i := 0; i < d.nComp; i++ { cs := d.tmp[1+2*i] // Component selector. - if cs != d.comps[i].c { + if cs != d.comp[i].c { return UnsupportedError("scan components out of order") } - scanComps[i].td = d.tmp[2+2*i] >> 4 - scanComps[i].ta = d.tmp[2+2*i] & 0x0f + scan[i].td = d.tmp[2+2*i] >> 4 + scan[i].ta = d.tmp[2+2*i] & 0x0f } // mxx and myy are the number of MCUs (Minimum Coded Units) in the image. - h0, v0 := d.comps[0].h, d.comps[0].v // The h and v values from the Y components. + h0, v0 := d.comp[0].h, d.comp[0].v // The h and v values from the Y components. mxx := (d.width + 8*h0 - 1) / (8 * h0) myy := (d.height + 8*v0 - 1) / (8 * v0) - if d.img == nil { - var subsampleRatio ycbcr.SubsampleRatio - n := h0 * v0 - switch n { - case 1: - subsampleRatio = ycbcr.SubsampleRatio444 - case 2: - subsampleRatio = ycbcr.SubsampleRatio422 - case 4: - subsampleRatio = ycbcr.SubsampleRatio420 - default: - panic("unreachable") - } - b := make([]byte, mxx*myy*(1*8*8*n+2*8*8)) - d.img = &ycbcr.YCbCr{ - Y: b[mxx*myy*(0*8*8*n+0*8*8) : mxx*myy*(1*8*8*n+0*8*8)], - Cb: b[mxx*myy*(1*8*8*n+0*8*8) : mxx*myy*(1*8*8*n+1*8*8)], - Cr: b[mxx*myy*(1*8*8*n+1*8*8) : mxx*myy*(1*8*8*n+2*8*8)], - SubsampleRatio: subsampleRatio, - YStride: mxx * 8 * h0, - CStride: mxx * 8, - Rect: image.Rect(0, 0, d.width, d.height), - } + if d.img1 == nil && d.img3 == nil { + d.makeImg(h0, v0, mxx, myy) } mcu, expectedRST := 0, uint8(rst0Marker) - var allZeroes block - var dc [nComponent]int + var ( + b block + dc [nColorComponent]int + ) for my := 0; my < myy; my++ { for mx := 0; mx < mxx; mx++ { - for i := 0; i < nComponent; i++ { - qt := &d.quant[d.comps[i].tq] - for j := 0; j < d.comps[i].h*d.comps[i].v; j++ { - d.blocks[i][j] = allZeroes + for i := 0; i < d.nComp; i++ { + qt := &d.quant[d.comp[i].tq] + for j := 0; j < d.comp[i].h*d.comp[i].v; j++ { + // TODO(nigeltao): make this a "var b block" once the compiler's escape + // analysis is good enough to allocate it on the stack, not the heap. + b = block{} // Decode the DC coefficient, as specified in section F.2.2.1. - value, err := d.decodeHuffman(&d.huff[dcTableClass][scanComps[i].td]) + value, err := d.decodeHuffman(&d.huff[dcTable][scan[i].td]) if err != nil { return err } @@ -297,11 +289,11 @@ func (d *decoder) processSOS(n int) os.Error { return err } dc[i] += dcDelta - d.blocks[i][j][0] = dc[i] * qt[0] + b[0] = dc[i] * qt[0] // Decode the AC coefficients, as specified in section F.2.2.2. for k := 1; k < blockSize; k++ { - value, err := d.decodeHuffman(&d.huff[acTableClass][scanComps[i].ta]) + value, err := d.decodeHuffman(&d.huff[acTable][scan[i].ta]) if err != nil { return err } @@ -316,7 +308,7 @@ func (d *decoder) processSOS(n int) os.Error { if err != nil { return err } - d.blocks[i][j][unzig[k]] = ac * qt[k] + b[unzig[k]] = ac * qt[k] } else { if val0 != 0x0f { break @@ -325,10 +317,32 @@ func (d *decoder) processSOS(n int) os.Error { } } - idct(&d.blocks[i][j]) + // Perform the inverse DCT and store the MCU component to the image. + if d.nComp == nGrayComponent { + idct(d.tmp[:64], 8, &b) + // Convert from []uint8 to []image.GrayColor. + p := d.img1.Pix[8*(my*d.img1.Stride+mx):] + for y := 0; y < 8; y++ { + dst := p[y*d.img1.Stride:] + src := d.tmp[8*y:] + for x := 0; x < 8; x++ { + dst[x] = image.GrayColor{src[x]} + } + } + } else { + switch i { + case 0: + mx0 := h0*mx + (j % 2) + my0 := v0*my + (j / 2) + idct(d.img3.Y[8*(my0*d.img3.YStride+mx0):], d.img3.YStride, &b) + case 1: + idct(d.img3.Cb[8*(my*d.img3.CStride+mx):], d.img3.CStride, &b) + case 2: + idct(d.img3.Cr[8*(my*d.img3.CStride+mx):], d.img3.CStride, &b) + } + } } // for j } // for i - d.storeMCU(mx, my) mcu++ if d.ri > 0 && mcu%d.ri == 0 && mcu < mxx*myy { // A more sophisticated decoder could use RST[0-7] markers to resynchronize from corrupt input, @@ -347,9 +361,7 @@ func (d *decoder) processSOS(n int) os.Error { // Reset the Huffman decoder. d.b = bits{} // Reset the DC components, as per section F.2.1.3.1. - for i := 0; i < nComponent; i++ { - dc[i] = 0 - } + dc = [nColorComponent]int{} } } // for mx } // for my @@ -437,7 +449,13 @@ func (d *decoder) decode(r io.Reader, configOnly bool) (image.Image, os.Error) { return nil, err } } - return d.img, nil + if d.img1 != nil { + return d.img1, nil + } + if d.img3 != nil { + return d.img3, nil + } + return nil, FormatError("missing SOS marker") } // Decode reads a JPEG image from r and returns it as an image.Image. @@ -453,7 +471,13 @@ func DecodeConfig(r io.Reader) (image.Config, os.Error) { if _, err := d.decode(r, true); err != nil { return image.Config{}, err } - return image.Config{image.RGBAColorModel, d.width, d.height}, nil + switch d.nComp { + case nGrayComponent: + return image.Config{image.GrayColorModel, d.width, d.height}, nil + case nColorComponent: + return image.Config{ycbcr.YCbCrColorModel, d.width, d.height}, nil + } + return image.Config{}, FormatError("missing SOF marker") } func init() { diff --git a/src/pkg/image/jpeg/writer.go b/src/pkg/image/jpeg/writer.go index 52b3dc4e2..eddaaefb6 100644 --- a/src/pkg/image/jpeg/writer.go +++ b/src/pkg/image/jpeg/writer.go @@ -221,8 +221,7 @@ type encoder struct { // buf is a scratch buffer. buf [16]byte // bits and nBits are accumulated bits to write to w. - bits uint32 - nBits uint8 + bits, nBits uint32 // quant is the scaled quantization tables. quant [nQuantIndex][blockSize]byte } @@ -250,7 +249,7 @@ func (e *encoder) writeByte(b byte) { // emit emits the least significant nBits bits of bits to the bitstream. // The precondition is bits < 1<<nBits && nBits <= 16. -func (e *encoder) emit(bits uint32, nBits uint8) { +func (e *encoder) emit(bits, nBits uint32) { nBits += e.nBits bits <<= 32 - nBits bits |= e.bits @@ -269,7 +268,7 @@ func (e *encoder) emit(bits uint32, nBits uint8) { // emitHuff emits the given value with the given Huffman encoder. func (e *encoder) emitHuff(h huffIndex, value int) { x := theHuffmanLUT[h][value] - e.emit(x&(1<<24-1), uint8(x>>24)) + e.emit(x&(1<<24-1), x>>24) } // emitHuffRLE emits a run of runLength copies of value encoded with the given @@ -279,11 +278,11 @@ func (e *encoder) emitHuffRLE(h huffIndex, runLength, value int) { if a < 0 { a, b = -value, value-1 } - var nBits uint8 + var nBits uint32 if a < 0x100 { - nBits = bitCount[a] + nBits = uint32(bitCount[a]) } else { - nBits = 8 + bitCount[a>>8] + nBits = 8 + uint32(bitCount[a>>8]) } e.emitHuff(h, runLength<<4|int(nBits)) if nBits > 0 { @@ -302,34 +301,31 @@ func (e *encoder) writeMarkerHeader(marker uint8, markerlen int) { // writeDQT writes the Define Quantization Table marker. func (e *encoder) writeDQT() { - markerlen := 2 - for _, q := range e.quant { - markerlen += 1 + len(q) - } + markerlen := 2 + int(nQuantIndex)*(1+blockSize) e.writeMarkerHeader(dqtMarker, markerlen) - for i, q := range e.quant { + for i := range e.quant { e.writeByte(uint8(i)) - e.write(q[:]) + e.write(e.quant[i][:]) } } // writeSOF0 writes the Start Of Frame (Baseline) marker. func (e *encoder) writeSOF0(size image.Point) { - markerlen := 8 + 3*nComponent + markerlen := 8 + 3*nColorComponent e.writeMarkerHeader(sof0Marker, markerlen) e.buf[0] = 8 // 8-bit color. e.buf[1] = uint8(size.Y >> 8) e.buf[2] = uint8(size.Y & 0xff) e.buf[3] = uint8(size.X >> 8) e.buf[4] = uint8(size.X & 0xff) - e.buf[5] = nComponent - for i := 0; i < nComponent; i++ { + e.buf[5] = nColorComponent + for i := 0; i < nColorComponent; i++ { e.buf[3*i+6] = uint8(i + 1) // We use 4:2:0 chroma subsampling. e.buf[3*i+7] = "\x22\x11\x11"[i] e.buf[3*i+8] = "\x00\x01\x01"[i] } - e.write(e.buf[:3*(nComponent-1)+9]) + e.write(e.buf[:3*(nColorComponent-1)+9]) } // writeDHT writes the Define Huffman Table marker. diff --git a/src/pkg/image/png/reader_test.go b/src/pkg/image/png/reader_test.go index efa6336d7..bcc1a3db4 100644 --- a/src/pkg/image/png/reader_test.go +++ b/src/pkg/image/png/reader_test.go @@ -28,6 +28,7 @@ var filenames = []string{ "basn3p02", "basn3p04", "basn3p08", + "basn3p08-trns", "basn4a08", "basn4a16", "basn6a08", @@ -98,17 +99,30 @@ func sng(w io.WriteCloser, filename string, png image.Image) { // (the PNG spec section 11.3 says "Ancillary chunks may be ignored by a decoder"). io.WriteString(w, "gAMA {1.0000}\n") - // Write the PLTE (if applicable). + // Write the PLTE and tRNS (if applicable). if cpm != nil { + lastAlpha := -1 io.WriteString(w, "PLTE {\n") - for i := 0; i < len(cpm); i++ { - r, g, b, _ := cpm[i].RGBA() + for i, c := range cpm { + r, g, b, a := c.RGBA() + if a != 0xffff { + lastAlpha = i + } r >>= 8 g >>= 8 b >>= 8 fmt.Fprintf(w, " (%3d,%3d,%3d) # rgb = (0x%02x,0x%02x,0x%02x)\n", r, g, b, r, g, b) } io.WriteString(w, "}\n") + if lastAlpha != -1 { + io.WriteString(w, "tRNS {\n") + for i := 0; i <= lastAlpha; i++ { + _, _, _, a := cpm[i].RGBA() + a >>= 8 + fmt.Fprintf(w, " %d", a) + } + io.WriteString(w, "}\n") + } } // Write the IMAGE. diff --git a/src/pkg/image/png/testdata/pngsuite/README b/src/pkg/image/png/testdata/pngsuite/README index abe3ecb20..c0f78bde8 100644 --- a/src/pkg/image/png/testdata/pngsuite/README +++ b/src/pkg/image/png/testdata/pngsuite/README @@ -10,6 +10,9 @@ The files basn0g01-30.png, basn0g02-29.png and basn0g04-31.png are in fact not part of pngsuite but were created from files in pngsuite. Their non-power- of-two sizes makes them useful for testing bit-depths smaller than a byte. +basn3a08.png was generated from basn6a08.png using the pngnq tool, which +converted it to the 8-bit paletted image with alpha values in tRNS chunk. + The *.sng files in this directory were generated from the *.png files by the sng command-line tool and some hand editing. The files basn0g0{1,2,4}.sng were actually generated by first converting the PNG diff --git a/src/pkg/image/png/testdata/pngsuite/basn3p08-trns.png b/src/pkg/image/png/testdata/pngsuite/basn3p08-trns.png Binary files differnew file mode 100644 index 000000000..b0fc0c1be --- /dev/null +++ b/src/pkg/image/png/testdata/pngsuite/basn3p08-trns.png diff --git a/src/pkg/image/png/testdata/pngsuite/basn3p08-trns.sng b/src/pkg/image/png/testdata/pngsuite/basn3p08-trns.sng new file mode 100644 index 000000000..78dc367bb --- /dev/null +++ b/src/pkg/image/png/testdata/pngsuite/basn3p08-trns.sng @@ -0,0 +1,301 @@ +#SNG: from basn3p08-trns.png +IHDR { + width: 32; height: 32; bitdepth: 8; + using color palette; +} +gAMA {1.0000} +PLTE { + (255, 3, 7) # rgb = (0xff,0x03,0x07) + (255, 4, 7) # rgb = (0xff,0x04,0x07) + (255, 9, 7) # rgb = (0xff,0x09,0x07) + (217, 14, 7) # rgb = (0xd9,0x0e,0x07) + (255, 14, 7) # rgb = (0xff,0x0e,0x07) + ( 2, 22, 19) # rgb = (0x02,0x16,0x13) + (255, 26, 7) # rgb = (0xff,0x1a,0x07) + (255, 31, 7) # rgb = (0xff,0x1f,0x07) + ( 10, 37, 14) # rgb = (0x0a,0x25,0x0e) + (179, 37, 6) # rgb = (0xb3,0x25,0x06) + (254, 42, 7) # rgb = (0xfe,0x2a,0x07) + (255, 45, 7) # rgb = (0xff,0x2d,0x07) + ( 25, 46, 9) # rgb = (0x19,0x2e,0x09) + ( 0, 48,254) # rgb = (0x00,0x30,0xfe) + ( 0, 48,255) # rgb = (0x00,0x30,0xff) + ( 0, 49,255) # rgb = (0x00,0x31,0xff) + ( 0, 51,254) # rgb = (0x00,0x33,0xfe) + ( 0, 52,255) # rgb = (0x00,0x34,0xff) + (255, 53, 7) # rgb = (0xff,0x35,0x07) + ( 0, 54,252) # rgb = (0x00,0x36,0xfc) + (254, 57, 7) # rgb = (0xfe,0x39,0x07) + (251, 57, 7) # rgb = (0xfb,0x39,0x07) + (247, 59, 7) # rgb = (0xf7,0x3b,0x07) + ( 0, 59, 61) # rgb = (0x00,0x3b,0x3d) + ( 0, 62,255) # rgb = (0x00,0x3e,0xff) + (142, 63, 5) # rgb = (0x8e,0x3f,0x05) + ( 0, 63,250) # rgb = (0x00,0x3f,0xfa) + (255, 63, 7) # rgb = (0xff,0x3f,0x07) + (253, 68, 7) # rgb = (0xfd,0x44,0x07) + ( 0, 73,255) # rgb = (0x00,0x49,0xff) + ( 0, 73,246) # rgb = (0x00,0x49,0xf6) + (255, 75, 7) # rgb = (0xff,0x4b,0x07) + ( 82, 85, 9) # rgb = (0x52,0x55,0x09) + (255, 85, 7) # rgb = (0xff,0x55,0x07) + ( 0, 89,255) # rgb = (0x00,0x59,0xff) + ( 0, 91,237) # rgb = (0x00,0x5b,0xed) + (255, 94, 7) # rgb = (0xff,0x5e,0x07) + (241,100, 7) # rgb = (0xf1,0x64,0x07) + ( 0,101,255) # rgb = (0x00,0x65,0xff) + (253,105, 7) # rgb = (0xfd,0x69,0x07) + ( 0,107,223) # rgb = (0x00,0x6b,0xdf) + (255,106, 7) # rgb = (0xff,0x6a,0x07) + ( 1,110, 95) # rgb = (0x01,0x6e,0x5f) + (255,115, 7) # rgb = (0xff,0x73,0x07) + ( 0,117,255) # rgb = (0x00,0x75,0xff) + (255,124, 7) # rgb = (0xff,0x7c,0x07) + (118,126, 10) # rgb = (0x76,0x7e,0x0a) + ( 0,130,250) # rgb = (0x00,0x82,0xfa) + ( 0,132,255) # rgb = (0x00,0x84,0xff) + ( 0,134,207) # rgb = (0x00,0x86,0xcf) + (255,134, 7) # rgb = (0xff,0x86,0x07) + ( 0,136,249) # rgb = (0x00,0x88,0xf9) + (219,140, 6) # rgb = (0xdb,0x8c,0x06) + ( 0,140,252) # rgb = (0x00,0x8c,0xfc) + ( 0,140,255) # rgb = (0x00,0x8c,0xff) + ( 1,142,136) # rgb = (0x01,0x8e,0x88) + (255,143, 7) # rgb = (0xff,0x8f,0x07) + (243,150, 7) # rgb = (0xf3,0x96,0x07) + (198,152, 7) # rgb = (0xc6,0x98,0x07) + (165,153, 7) # rgb = (0xa5,0x99,0x07) + ( 0,157,255) # rgb = (0x00,0x9d,0xff) + (255,158, 7) # rgb = (0xff,0x9e,0x07) + ( 70,159, 4) # rgb = (0x46,0x9f,0x04) + ( 0,160,251) # rgb = (0x00,0xa0,0xfb) + (203,163, 6) # rgb = (0xcb,0xa3,0x06) + ( 0,163,239) # rgb = (0x00,0xa3,0xef) + ( 1,164,178) # rgb = (0x01,0xa4,0xb2) + (255,166, 7) # rgb = (0xff,0xa6,0x07) + ( 1,169,165) # rgb = (0x01,0xa9,0xa5) + ( 1,170,255) # rgb = (0x01,0xaa,0xff) + (232,172, 6) # rgb = (0xe8,0xac,0x06) + (255,175, 7) # rgb = (0xff,0xaf,0x07) + (185,176,131) # rgb = (0xb9,0xb0,0x83) + ( 1,179,225) # rgb = (0x01,0xb3,0xe1) + (188,179,118) # rgb = (0xbc,0xb3,0x76) + (199,180, 6) # rgb = (0xc7,0xb4,0x06) + ( 1,182,255) # rgb = (0x01,0xb6,0xff) + ( 1,184,249) # rgb = (0x01,0xb8,0xf9) + (255,184, 7) # rgb = (0xff,0xb8,0x07) + (207,186, 71) # rgb = (0xcf,0xba,0x47) + (193,187, 6) # rgb = (0xc1,0xbb,0x06) + (253,191, 7) # rgb = (0xfd,0xbf,0x07) + (218,193, 48) # rgb = (0xda,0xc1,0x30) + ( 1,193,157) # rgb = (0x01,0xc1,0x9d) + ( 1,196,244) # rgb = (0x01,0xc4,0xf4) + ( 1,196,254) # rgb = (0x01,0xc4,0xfe) + ( 48,199, 3) # rgb = (0x30,0xc7,0x03) + (164,199, 5) # rgb = (0xa4,0xc7,0x05) + (220,202, 6) # rgb = (0xdc,0xca,0x06) + (253,203, 7) # rgb = (0xfd,0xcb,0x07) + ( 1,204,204) # rgb = (0x01,0xcc,0xcc) + (251,209, 7) # rgb = (0xfb,0xd1,0x07) + (231,208, 24) # rgb = (0xe7,0xd0,0x18) + ( 1,210,254) # rgb = (0x01,0xd2,0xfe) + ( 2,211,146) # rgb = (0x02,0xd3,0x92) + ( 1,212,156) # rgb = (0x01,0xd4,0x9c) + ( 1,213,252) # rgb = (0x01,0xd5,0xfc) + (237,219, 15) # rgb = (0xed,0xdb,0x0f) + ( 1,218,240) # rgb = (0x01,0xda,0xf0) + (165,220, 5) # rgb = (0xa5,0xdc,0x05) + ( 1,221,250) # rgb = (0x01,0xdd,0xfa) + (249,221, 6) # rgb = (0xf9,0xdd,0x06) + (146,222, 4) # rgb = (0x92,0xde,0x04) + ( 1,224,184) # rgb = (0x01,0xe0,0xb8) + ( 2,224,155) # rgb = (0x02,0xe0,0x9b) + (244,225, 10) # rgb = (0xf4,0xe1,0x0a) + (249,227, 7) # rgb = (0xf9,0xe3,0x07) + ( 2,229,133) # rgb = (0x02,0xe5,0x85) + (192,228, 6) # rgb = (0xc0,0xe4,0x06) + ( 37,230, 3) # rgb = (0x25,0xe6,0x03) + (246,230, 7) # rgb = (0xf6,0xe6,0x07) + (143,232, 4) # rgb = (0x8f,0xe8,0x04) + (244,233, 8) # rgb = (0xf4,0xe9,0x08) + ( 2,236,139) # rgb = (0x02,0xec,0x8b) + ( 1,236,227) # rgb = (0x01,0xec,0xe3) + ( 1,238,238) # rgb = (0x01,0xee,0xee) + (101,241, 4) # rgb = (0x65,0xf1,0x04) + ( 1,241,218) # rgb = (0x01,0xf1,0xda) + ( 1,240,232) # rgb = (0x01,0xf0,0xe8) + (167,240, 5) # rgb = (0xa7,0xf0,0x05) + ( 27,243, 2) # rgb = (0x1b,0xf3,0x02) + (126,243, 4) # rgb = (0x7e,0xf3,0x04) + ( 2,246,113) # rgb = (0x02,0xf6,0x71) + (133,248, 5) # rgb = (0x85,0xf8,0x05) + ( 22,250, 1) # rgb = (0x16,0xfa,0x01) + ( 2,249,219) # rgb = (0x02,0xf9,0xdb) + (148,250, 5) # rgb = (0x94,0xfa,0x05) + ( 2,250,199) # rgb = (0x02,0xfa,0xc7) + (183,252, 5) # rgb = (0xb7,0xfc,0x05) + (176,252, 5) # rgb = (0xb0,0xfc,0x05) + ( 2,252,211) # rgb = (0x02,0xfc,0xd3) + ( 2,252,190) # rgb = (0x02,0xfc,0xbe) + (164,251, 5) # rgb = (0xa4,0xfb,0x05) + ( 12,254,128) # rgb = (0x0c,0xfe,0x80) + (192,253, 5) # rgb = (0xc0,0xfd,0x05) + (164,253, 5) # rgb = (0xa4,0xfd,0x05) + ( 26,254, 85) # rgb = (0x1a,0xfe,0x55) + ( 14,254, 1) # rgb = (0x0e,0xfe,0x01) + (133,253, 5) # rgb = (0x85,0xfd,0x05) + ( 4,253,180) # rgb = (0x04,0xfd,0xb4) + (196,253, 5) # rgb = (0xc4,0xfd,0x05) + ( 2,253,198) # rgb = (0x02,0xfd,0xc6) + ( 3,255, 91) # rgb = (0x03,0xff,0x5b) + ( 3,255, 80) # rgb = (0x03,0xff,0x50) + (186,255, 5) # rgb = (0xba,0xff,0x05) + ( 9,255, 2) # rgb = (0x09,0xff,0x02) + ( 3,255,118) # rgb = (0x03,0xff,0x76) + ( 9,255, 3) # rgb = (0x09,0xff,0x03) + ( 10,255, 1) # rgb = (0x0a,0xff,0x01) + ( 3,255, 76) # rgb = (0x03,0xff,0x4c) + ( 3,255, 86) # rgb = (0x03,0xff,0x56) + ( 3,255, 82) # rgb = (0x03,0xff,0x52) + ( 13,255, 1) # rgb = (0x0d,0xff,0x01) + ( 3,255, 49) # rgb = (0x03,0xff,0x31) + ( 3,255,101) # rgb = (0x03,0xff,0x65) + ( 61,255, 32) # rgb = (0x3d,0xff,0x20) + (129,255, 5) # rgb = (0x81,0xff,0x05) + (177,255, 5) # rgb = (0xb1,0xff,0x05) + ( 3,255, 37) # rgb = (0x03,0xff,0x25) + (149,255, 5) # rgb = (0x95,0xff,0x05) + ( 7,255, 6) # rgb = (0x07,0xff,0x06) + (192,255, 5) # rgb = (0xc0,0xff,0x05) + ( 2,255,131) # rgb = (0x02,0xff,0x83) + ( 3,255, 98) # rgb = (0x03,0xff,0x62) + ( 85,255, 11) # rgb = (0x55,0xff,0x0b) + ( 2,255,163) # rgb = (0x02,0xff,0xa3) + ( 2,255,149) # rgb = (0x02,0xff,0x95) + ( 4,255, 23) # rgb = (0x04,0xff,0x17) + ( 6,255, 12) # rgb = (0x06,0xff,0x0c) + ( 3,255, 67) # rgb = (0x03,0xff,0x43) + (160,255, 5) # rgb = (0xa0,0xff,0x05) + (119,255, 6) # rgb = (0x77,0xff,0x06) + (102,255, 8) # rgb = (0x66,0xff,0x08) + (255,255,255) # rgb = (0xff,0xff,0xff) + (254,254,254) # rgb = (0xfe,0xfe,0xfe) + (254,254,254) # rgb = (0xfe,0xfe,0xfe) + (252,252,252) # rgb = (0xfc,0xfc,0xfc) + (252,252,252) # rgb = (0xfc,0xfc,0xfc) + (250,250,250) # rgb = (0xfa,0xfa,0xfa) + (250,250,250) # rgb = (0xfa,0xfa,0xfa) + (248,248,248) # rgb = (0xf8,0xf8,0xf8) + (248,248,248) # rgb = (0xf8,0xf8,0xf8) + (247,247,247) # rgb = (0xf7,0xf7,0xf7) + (245,245,245) # rgb = (0xf5,0xf5,0xf5) + (245,245,245) # rgb = (0xf5,0xf5,0xf5) + (243,243,243) # rgb = (0xf3,0xf3,0xf3) + (243,243,243) # rgb = (0xf3,0xf3,0xf3) + (241,241,241) # rgb = (0xf1,0xf1,0xf1) + (241,241,241) # rgb = (0xf1,0xf1,0xf1) + (239,239,239) # rgb = (0xef,0xef,0xef) + (238,238,238) # rgb = (0xee,0xee,0xee) + (238,238,238) # rgb = (0xee,0xee,0xee) + (236,236,236) # rgb = (0xec,0xec,0xec) + (236,236,236) # rgb = (0xec,0xec,0xec) + (234,234,234) # rgb = (0xea,0xea,0xea) + (234,234,234) # rgb = (0xea,0xea,0xea) + (232,232,232) # rgb = (0xe8,0xe8,0xe8) + (231,231,231) # rgb = (0xe7,0xe7,0xe7) + (231,231,231) # rgb = (0xe7,0xe7,0xe7) + (229,229,229) # rgb = (0xe5,0xe5,0xe5) + (229,229,229) # rgb = (0xe5,0xe5,0xe5) + (227,227,227) # rgb = (0xe3,0xe3,0xe3) + (226,226,226) # rgb = (0xe2,0xe2,0xe2) + (226,226,226) # rgb = (0xe2,0xe2,0xe2) + (224,224,224) # rgb = (0xe0,0xe0,0xe0) + (224,224,224) # rgb = (0xe0,0xe0,0xe0) + (222,222,222) # rgb = (0xde,0xde,0xde) + (222,222,222) # rgb = (0xde,0xde,0xde) + (220,220,220) # rgb = (0xdc,0xdc,0xdc) + (219,219,219) # rgb = (0xdb,0xdb,0xdb) + (219,219,219) # rgb = (0xdb,0xdb,0xdb) + (217,217,217) # rgb = (0xd9,0xd9,0xd9) + (217,217,217) # rgb = (0xd9,0xd9,0xd9) + (215,215,215) # rgb = (0xd7,0xd7,0xd7) + (214,214,214) # rgb = (0xd6,0xd6,0xd6) + (214,214,214) # rgb = (0xd6,0xd6,0xd6) + (212,212,212) # rgb = (0xd4,0xd4,0xd4) + (212,212,212) # rgb = (0xd4,0xd4,0xd4) + (210,210,210) # rgb = (0xd2,0xd2,0xd2) + (209,209,209) # rgb = (0xd1,0xd1,0xd1) + (209,209,209) # rgb = (0xd1,0xd1,0xd1) + (207,207,207) # rgb = (0xcf,0xcf,0xcf) + (205,205,205) # rgb = (0xcd,0xcd,0xcd) + (205,205,205) # rgb = (0xcd,0xcd,0xcd) + (204,204,204) # rgb = (0xcc,0xcc,0xcc) + (204,204,204) # rgb = (0xcc,0xcc,0xcc) + (202,202,202) # rgb = (0xca,0xca,0xca) + (201,201,201) # rgb = (0xc9,0xc9,0xc9) + (201,201,201) # rgb = (0xc9,0xc9,0xc9) + (199,199,199) # rgb = (0xc7,0xc7,0xc7) + (199,199,199) # rgb = (0xc7,0xc7,0xc7) + (197,197,197) # rgb = (0xc5,0xc5,0xc5) + (196,196,196) # rgb = (0xc4,0xc4,0xc4) + (196,196,196) # rgb = (0xc4,0xc4,0xc4) + (194,194,194) # rgb = (0xc2,0xc2,0xc2) + (193,193,193) # rgb = (0xc1,0xc1,0xc1) + (193,193,193) # rgb = (0xc1,0xc1,0xc1) + (191,191,191) # rgb = (0xbf,0xbf,0xbf) + (191,191,191) # rgb = (0xbf,0xbf,0xbf) + (189,189,189) # rgb = (0xbd,0xbd,0xbd) + (188,188,188) # rgb = (0xbc,0xbc,0xbc) + (188,188,188) # rgb = (0xbc,0xbc,0xbc) + (186,186,186) # rgb = (0xba,0xba,0xba) + (185,185,185) # rgb = (0xb9,0xb9,0xb9) + (185,185,185) # rgb = (0xb9,0xb9,0xb9) + (183,183,183) # rgb = (0xb7,0xb7,0xb7) + (182,182,182) # rgb = (0xb6,0xb6,0xb6) + (182,182,182) # rgb = (0xb6,0xb6,0xb6) + (180,180,180) # rgb = (0xb4,0xb4,0xb4) + (178,178,178) # rgb = (0xb2,0xb2,0xb2) + (178,178,178) # rgb = (0xb2,0xb2,0xb2) + (177,177,177) # rgb = (0xb1,0xb1,0xb1) + (177,177,177) # rgb = (0xb1,0xb1,0xb1) + (175,175,175) # rgb = (0xaf,0xaf,0xaf) + (174,174,174) # rgb = (0xae,0xae,0xae) + (174,174,174) # rgb = (0xae,0xae,0xae) +} +tRNS { + 197 187 190 194 186 4 186 189 4 195 84 191 5 193 175 163 205 150 191 213 88 75 67 8 147 191 220 203 95 151 223 199 8 207 156 227 199 65 163 98 226 204 12 202 167 201 11 65 178 228 205 74 59 87 178 19 201 99 18 14 184 204 184 96 22 61 227 199 22 193 97 197 254 59 253 28 192 102 199 247 58 198 244 30 109 202 188 32 96 196 60 203 239 202 230 41 207 237 119 53 213 209 37 55 45 230 214 233 92 185 223 50 230 57 124 217 43 133 221 95 198 47 233 99 194 221 107 138 152 144 226 140 133 220 172 125 218 196 118 225 161 223 235 238 200 155 147 146 172 236 236 151 183 150 234 216 217 211 151 219 132 185 145 147 217 138 144 137 142 151 217 217 213} +IMAGE { + pixels hex +0520201616160a0a0a0a0a0a0a0a010101010101010101000000000000000000 +053a3a161616160a0a0a0a0a0a0a0a0a0a06060606060607070707070707071b +053a3a3a161616161615151c1c1c1c1c1c1c12121212121b1b1b1b1b1b1b1b1b +053a3a3a3a252525252527272727272727272724242424242424212121212121 +053a3a3a4034343425252727272727393939392d2d2d2d2d2d2d323232323232 +053a3a404034343434343939393939393939394747474343433d3d3d3d3d3d3d +053a404b4b4b50505046464646464646464659595959595151514e5b5b616161 +053a404b4b4b50505058585858585858588c8c8c595959595b656a6e70707070 +053a4b4b4b4b5050506c5858585858588c8c8c8c8c8c5965656a6a6e70707070 +053b4b4b4b636363506c6c6c6c6c6c8781808c8c8c86a1a1a1906e6e70707070 +053b4b5757636363636c6c6c6c7787878181808c8c86a1a190909d9d9d9daa70 +053b576666666f6363777777777e8787848481808086a19090aaaaaaaa9f9f9f +053b576666797979797b7b7b7b7b8a8a8a8a848480809c9c9c9c9c9c9c9c9c9c +053b66747474747474747b7b7b7b8a8a8a8a8a8aacacacacacacacacacaca4a4 +052e7474747474747474747b7b7b8a8a8a6d6d6d6d6d6d6da4a4a4a4a4a4a4a4 +052e7474747474747474a0a0a0a0a0a09393936d6d6d6d787878787878787878 +05207474747474a0a0a0a0a0a0a0a0a093939191949494948989898989898989 +052a2a2a7171717171a7a7a7a7a7a7a7a7a79e9e9e9e9e9e9e9e959595959595 +052a53536871717171717171a9a9a9a9a9a9a9a9a9a9a9a99595959595959595 +053753536871717171717171a3a3a3a3a3a3a3a3979797979a9a9a9a8e8e8e8e +05445353686871717171717171a5a2a2a2a2a2929292928585857a7a7a7a7a7a +054453535f68687171717171a5a5a5a5a5a5a6a6a6a6a68b8b8b8b8b8b8b8b6b +054444535f686767676767677272727f7f8383838383838d8d8d8d8d8d8d8b8b +054444535f6767675a5a5a627272727275757f7f7f7f5d73737d7d7d82828282 +0544445367675a5a5a5a4d546262727272757575755d5d5d7373737376767676 +054444535349495a5a5a4d4d54626262626275754c5d5d5d5d60646464767676 +054444444949494949494d4d4d5454546262624c4c4c4c4c5555556060646464 +05444444444941414133353f3f3f3f3f3f4d3636363c3c454545454531313131 +05444444442f2f2f2f333535353535352c2c2c2c2c3030303030282828282828 +053744442f2f2f2f2f2f333535351d1d22222222262626262323232323232323 +053737372f2f2f2f2f2f2f331818181818181d1d1d1d1d131a1a1a1a1a1e1e1e +052a37372f2f2f2f2f2f18111111110f0e0e0e0e0d0d0d0d0d0d0d0d0d0d0d13 +} diff --git a/src/pkg/image/png/writer.go b/src/pkg/image/png/writer.go index 2d593f6a7..d770cfad5 100644 --- a/src/pkg/image/png/writer.go +++ b/src/pkg/image/png/writer.go @@ -130,12 +130,8 @@ func (e *encoder) writePLTE(p image.PalettedColorModel) { e.err = FormatError("bad palette length: " + strconv.Itoa(len(p))) return } - for i := 0; i < len(p); i++ { - r, g, b, a := p[i].RGBA() - if a != 0xffff { - e.err = UnsupportedError("non-opaque palette color") - return - } + for i, c := range p { + r, g, b, _ := c.RGBA() e.tmp[3*i+0] = uint8(r >> 8) e.tmp[3*i+1] = uint8(g >> 8) e.tmp[3*i+2] = uint8(b >> 8) @@ -143,6 +139,21 @@ func (e *encoder) writePLTE(p image.PalettedColorModel) { e.writeChunk(e.tmp[0:3*len(p)], "PLTE") } +func (e *encoder) maybeWritetRNS(p image.PalettedColorModel) { + last := -1 + for i, c := range p { + _, _, _, a := c.RGBA() + if a != 0xffff { + last = i + } + e.tmp[i] = uint8(a >> 8) + } + if last == -1 { + return + } + e.writeChunk(e.tmp[:last+1], "tRNS") +} + // An encoder is an io.Writer that satisfies writes by writing PNG IDAT chunks, // including an 8-byte header and 4-byte CRC checksum per Write call. Such calls // should be relatively infrequent, since writeIDATs uses a bufio.Writer. @@ -163,7 +174,7 @@ func (e *encoder) Write(b []byte) (int, os.Error) { // Chooses the filter to use for encoding the current row, and applies it. // The return value is the index of the filter and also of the row in cr that has had it applied. -func filter(cr [][]byte, pr []byte, bpp int) int { +func filter(cr *[nFilter][]byte, pr []byte, bpp int) int { // We try all five filter types, and pick the one that minimizes the sum of absolute differences. // This is the same heuristic that libpng uses, although the filters are attempted in order of // estimated most likely to be minimal (ftUp, ftPaeth, ftNone, ftSub, ftAverage), rather than @@ -293,7 +304,7 @@ func writeImage(w io.Writer, m image.Image, cb int) os.Error { // The +1 is for the per-row filter type, which is at cr[*][0]. b := m.Bounds() var cr [nFilter][]uint8 - for i := 0; i < len(cr); i++ { + for i := range cr { cr[i] = make([]uint8, 1+bpp*b.Dx()) cr[i][0] = uint8(i) } @@ -301,78 +312,84 @@ func writeImage(w io.Writer, m image.Image, cb int) os.Error { for y := b.Min.Y; y < b.Max.Y; y++ { // Convert from colors to bytes. + i := 1 switch cb { case cbG8: for x := b.Min.X; x < b.Max.X; x++ { c := image.GrayColorModel.Convert(m.At(x, y)).(image.GrayColor) - cr[0][x+1] = c.Y + cr[0][i] = c.Y + i++ } case cbTC8: // We have previously verified that the alpha value is fully opaque. cr0 := cr[0] if rgba != nil { yoff := y * rgba.Stride - xoff := 3*b.Min.X + 1 for _, color := range rgba.Pix[yoff+b.Min.X : yoff+b.Max.X] { - cr0[xoff] = color.R - cr0[xoff+1] = color.G - cr0[xoff+2] = color.B - xoff += 3 + cr0[i+0] = color.R + cr0[i+1] = color.G + cr0[i+2] = color.B + i += 3 } } else { for x := b.Min.X; x < b.Max.X; x++ { r, g, b, _ := m.At(x, y).RGBA() - cr0[3*x+1] = uint8(r >> 8) - cr0[3*x+2] = uint8(g >> 8) - cr0[3*x+3] = uint8(b >> 8) + cr0[i+0] = uint8(r >> 8) + cr0[i+1] = uint8(g >> 8) + cr0[i+2] = uint8(b >> 8) + i += 3 } } case cbP8: rowOffset := y * paletted.Stride - copy(cr[0][b.Min.X+1:], paletted.Pix[rowOffset+b.Min.X:rowOffset+b.Max.X]) + copy(cr[0][1:], paletted.Pix[rowOffset+b.Min.X:rowOffset+b.Max.X]) case cbTCA8: // Convert from image.Image (which is alpha-premultiplied) to PNG's non-alpha-premultiplied. for x := b.Min.X; x < b.Max.X; x++ { c := image.NRGBAColorModel.Convert(m.At(x, y)).(image.NRGBAColor) - cr[0][4*x+1] = c.R - cr[0][4*x+2] = c.G - cr[0][4*x+3] = c.B - cr[0][4*x+4] = c.A + cr[0][i+0] = c.R + cr[0][i+1] = c.G + cr[0][i+2] = c.B + cr[0][i+3] = c.A + i += 4 } case cbG16: for x := b.Min.X; x < b.Max.X; x++ { c := image.Gray16ColorModel.Convert(m.At(x, y)).(image.Gray16Color) - cr[0][2*x+1] = uint8(c.Y >> 8) - cr[0][2*x+2] = uint8(c.Y) + cr[0][i+0] = uint8(c.Y >> 8) + cr[0][i+1] = uint8(c.Y) + i += 2 } case cbTC16: + // We have previously verified that the alpha value is fully opaque. for x := b.Min.X; x < b.Max.X; x++ { - // We have previously verified that the alpha value is fully opaque. r, g, b, _ := m.At(x, y).RGBA() - cr[0][6*x+1] = uint8(r >> 8) - cr[0][6*x+2] = uint8(r) - cr[0][6*x+3] = uint8(g >> 8) - cr[0][6*x+4] = uint8(g) - cr[0][6*x+5] = uint8(b >> 8) - cr[0][6*x+6] = uint8(b) + cr[0][i+0] = uint8(r >> 8) + cr[0][i+1] = uint8(r) + cr[0][i+2] = uint8(g >> 8) + cr[0][i+3] = uint8(g) + cr[0][i+4] = uint8(b >> 8) + cr[0][i+5] = uint8(b) + i += 6 } case cbTCA16: // Convert from image.Image (which is alpha-premultiplied) to PNG's non-alpha-premultiplied. for x := b.Min.X; x < b.Max.X; x++ { c := image.NRGBA64ColorModel.Convert(m.At(x, y)).(image.NRGBA64Color) - cr[0][8*x+1] = uint8(c.R >> 8) - cr[0][8*x+2] = uint8(c.R) - cr[0][8*x+3] = uint8(c.G >> 8) - cr[0][8*x+4] = uint8(c.G) - cr[0][8*x+5] = uint8(c.B >> 8) - cr[0][8*x+6] = uint8(c.B) - cr[0][8*x+7] = uint8(c.A >> 8) - cr[0][8*x+8] = uint8(c.A) + cr[0][i+0] = uint8(c.R >> 8) + cr[0][i+1] = uint8(c.R) + cr[0][i+2] = uint8(c.G >> 8) + cr[0][i+3] = uint8(c.G) + cr[0][i+4] = uint8(c.B >> 8) + cr[0][i+5] = uint8(c.B) + cr[0][i+6] = uint8(c.A >> 8) + cr[0][i+7] = uint8(c.A) + i += 8 } } // Apply the filter. - f := filter(cr[0:nFilter], pr, bpp) + f := filter(&cr, pr, bpp) // Write the compressed bytes. _, err = zw.Write(cr[f]) @@ -447,6 +464,7 @@ func Encode(w io.Writer, m image.Image) os.Error { e.writeIHDR() if pal != nil { e.writePLTE(pal.Palette) + e.maybeWritetRNS(pal.Palette) } e.writeIDATs() e.writeIEND() diff --git a/src/pkg/image/png/writer_test.go b/src/pkg/image/png/writer_test.go index 6b054aaa8..271519a11 100644 --- a/src/pkg/image/png/writer_test.go +++ b/src/pkg/image/png/writer_test.go @@ -5,9 +5,9 @@ package png import ( + "bytes" "fmt" "image" - "io" "io/ioutil" "os" "testing" @@ -15,21 +15,38 @@ import ( func diff(m0, m1 image.Image) os.Error { b0, b1 := m0.Bounds(), m1.Bounds() - if !b0.Eq(b1) { + if !b0.Size().Eq(b1.Size()) { return fmt.Errorf("dimensions differ: %v vs %v", b0, b1) } + dx := b1.Min.X - b0.Min.X + dy := b1.Min.Y - b0.Min.Y for y := b0.Min.Y; y < b0.Max.Y; y++ { for x := b0.Min.X; x < b0.Max.X; x++ { - r0, g0, b0, a0 := m0.At(x, y).RGBA() - r1, g1, b1, a1 := m1.At(x, y).RGBA() + c0 := m0.At(x, y) + c1 := m1.At(x+dx, y+dy) + r0, g0, b0, a0 := c0.RGBA() + r1, g1, b1, a1 := c1.RGBA() if r0 != r1 || g0 != g1 || b0 != b1 || a0 != a1 { - return fmt.Errorf("colors differ at (%d, %d): %v vs %v", x, y, m0.At(x, y), m1.At(x, y)) + return fmt.Errorf("colors differ at (%d, %d): %v vs %v", x, y, c0, c1) } } } return nil } +func encodeDecode(m image.Image) (image.Image, os.Error) { + b := bytes.NewBuffer(nil) + err := Encode(b, m) + if err != nil { + return nil, err + } + m, err = Decode(b) + if err != nil { + return nil, err + } + return m, nil +} + func TestWriter(t *testing.T) { // The filenames variable is declared in reader_test.go. names := filenames @@ -44,26 +61,16 @@ func TestWriter(t *testing.T) { t.Error(fn, err) continue } - // Read the image again, and push it through a pipe that encodes at the write end, and decodes at the read end. - pr, pw := io.Pipe() - defer pr.Close() - go func() { - defer pw.Close() - m1, err := readPng(qfn) - if err != nil { - t.Error(fn, err) - return - } - err = Encode(pw, m1) - if err != nil { - t.Error(fn, err) - return - } - }() - m2, err := Decode(pr) + // Read the image again, encode it, and decode it. + m1, err := readPng(qfn) if err != nil { t.Error(fn, err) - continue + return + } + m2, err := encodeDecode(m1) + if err != nil { + t.Error(fn, err) + return } // Compare the two. err = diff(m0, m2) @@ -74,6 +81,26 @@ func TestWriter(t *testing.T) { } } +func TestSubimage(t *testing.T) { + m0 := image.NewRGBA(256, 256) + for y := 0; y < 256; y++ { + for x := 0; x < 256; x++ { + m0.Set(x, y, image.RGBAColor{uint8(x), uint8(y), 0, 255}) + } + } + m0.Rect = image.Rect(50, 30, 250, 130) + m1, err := encodeDecode(m0) + if err != nil { + t.Error(err) + return + } + err = diff(m0, m1) + if err != nil { + t.Error(err) + return + } +} + func BenchmarkEncodePaletted(b *testing.B) { b.StopTimer() img := image.NewPaletted(640, 480, diff --git a/src/pkg/image/testdata/video-001.5bpp.gif b/src/pkg/image/testdata/video-001.5bpp.gif Binary files differnew file mode 100644 index 000000000..ce53104b2 --- /dev/null +++ b/src/pkg/image/testdata/video-001.5bpp.gif diff --git a/src/pkg/image/testdata/video-001.interlaced.gif b/src/pkg/image/testdata/video-001.interlaced.gif Binary files differnew file mode 100644 index 000000000..590594ea9 --- /dev/null +++ b/src/pkg/image/testdata/video-001.interlaced.gif diff --git a/src/pkg/image/testdata/video-005.gray.jpeg b/src/pkg/image/testdata/video-005.gray.jpeg Binary files differnew file mode 100644 index 000000000..f9d6e5cdb --- /dev/null +++ b/src/pkg/image/testdata/video-005.gray.jpeg diff --git a/src/pkg/image/testdata/video-005.gray.png b/src/pkg/image/testdata/video-005.gray.png Binary files differnew file mode 100644 index 000000000..0b0ee7538 --- /dev/null +++ b/src/pkg/image/testdata/video-005.gray.png diff --git a/src/pkg/image/tiff/Makefile b/src/pkg/image/tiff/Makefile new file mode 100644 index 000000000..1a001afb9 --- /dev/null +++ b/src/pkg/image/tiff/Makefile @@ -0,0 +1,13 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=image/tiff +GOFILES=\ + buffer.go\ + consts.go\ + reader.go\ + +include ../../../Make.pkg diff --git a/src/pkg/image/tiff/buffer.go b/src/pkg/image/tiff/buffer.go new file mode 100644 index 000000000..7c0714225 --- /dev/null +++ b/src/pkg/image/tiff/buffer.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tiff + +import ( + "io" + "os" +) + +// buffer buffers an io.Reader to satisfy io.ReaderAt. +type buffer struct { + r io.Reader + buf []byte +} + +func (b *buffer) ReadAt(p []byte, off int64) (int, os.Error) { + o := int(off) + end := o + len(p) + if int64(end) != off+int64(len(p)) { + return 0, os.EINVAL + } + + m := len(b.buf) + if end > m { + if end > cap(b.buf) { + newcap := 1024 + for newcap < end { + newcap *= 2 + } + newbuf := make([]byte, end, newcap) + copy(newbuf, b.buf) + b.buf = newbuf + } else { + b.buf = b.buf[:end] + } + if n, err := io.ReadFull(b.r, b.buf[m:end]); err != nil { + end = m + n + b.buf = b.buf[:end] + return copy(p, b.buf[o:end]), err + } + } + + return copy(p, b.buf[o:end]), nil +} + +// newReaderAt converts an io.Reader into an io.ReaderAt. +func newReaderAt(r io.Reader) io.ReaderAt { + if ra, ok := r.(io.ReaderAt); ok { + return ra + } + return &buffer{ + r: r, + buf: make([]byte, 0, 1024), + } +} diff --git a/src/pkg/image/tiff/buffer_test.go b/src/pkg/image/tiff/buffer_test.go new file mode 100644 index 000000000..4f3e68e83 --- /dev/null +++ b/src/pkg/image/tiff/buffer_test.go @@ -0,0 +1,36 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tiff + +import ( + "os" + "strings" + "testing" +) + +var readAtTests = []struct { + n int + off int64 + s string + err os.Error +}{ + {2, 0, "ab", nil}, + {6, 0, "abcdef", nil}, + {3, 3, "def", nil}, + {3, 5, "f", os.EOF}, + {3, 6, "", os.EOF}, +} + +func TestReadAt(t *testing.T) { + r := newReaderAt(strings.NewReader("abcdef")) + b := make([]byte, 10) + for _, test := range readAtTests { + n, err := r.ReadAt(b[:test.n], test.off) + s := string(b[:n]) + if s != test.s || err != test.err { + t.Errorf("buffer.ReadAt(<%v bytes>, %v): got %v, %q; want %v, %q", test.n, test.off, err, s, test.err, test.s) + } + } +} diff --git a/src/pkg/image/tiff/consts.go b/src/pkg/image/tiff/consts.go new file mode 100644 index 000000000..169ba2772 --- /dev/null +++ b/src/pkg/image/tiff/consts.go @@ -0,0 +1,103 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tiff + +// A tiff image file contains one or more images. The metadata +// of each image is contained in an Image File Directory (IFD), +// which contains entries of 12 bytes each and is described +// on page 14-16 of the specification. An IFD entry consists of +// +// - a tag, which describes the signification of the entry, +// - the data type and length of the entry, +// - the data itself or a pointer to it if it is more than 4 bytes. +// +// The presence of a length means that each IFD is effectively an array. + +const ( + leHeader = "II\x2A\x00" // Header for little-endian files. + beHeader = "MM\x00\x2A" // Header for big-endian files. + + ifdLen = 12 // Length of an IFD entry in bytes. +) + +// Data types (p. 14-16 of the spec). +const ( + dtByte = 1 + dtASCII = 2 + dtShort = 3 + dtLong = 4 + dtRational = 5 +) + +// The length of one instance of each data type in bytes. +var lengths = [...]uint32{0, 1, 1, 2, 4, 8} + +// Tags (see p. 28-41 of the spec). +const ( + tImageWidth = 256 + tImageLength = 257 + tBitsPerSample = 258 + tCompression = 259 + tPhotometricInterpretation = 262 + + tStripOffsets = 273 + tSamplesPerPixel = 277 + tRowsPerStrip = 278 + tStripByteCounts = 279 + + tXResolution = 282 + tYResolution = 283 + tResolutionUnit = 296 + + tPredictor = 317 + tColorMap = 320 + tExtraSamples = 338 + tSampleFormat = 339 +) + +// Compression types (defined in various places in the spec and supplements). +const ( + cNone = 1 + cCCITT = 2 + cG3 = 3 // Group 3 Fax. + cG4 = 4 // Group 4 Fax. + cLZW = 5 + cJPEGOld = 6 // Superseded by cJPEG. + cJPEG = 7 + cDeflate = 8 // zlib compression. + cPackBits = 32773 + cDeflateOld = 32946 // Superseded by cDeflate. +) + +// Photometric interpretation values (see p. 37 of the spec). +const ( + pWhiteIsZero = 0 + pBlackIsZero = 1 + pRGB = 2 + pPaletted = 3 + pTransMask = 4 // transparency mask + pCMYK = 5 + pYCbCr = 6 + pCIELab = 8 +) + +// Values for the tPredictor tag (page 64-65 of the spec). +const ( + prNone = 1 + prHorizontal = 2 +) + +// imageMode represents the mode of the image. +type imageMode int + +const ( + mBilevel imageMode = iota + mPaletted + mGray + mGrayInvert + mRGB + mRGBA + mNRGBA +) diff --git a/src/pkg/image/tiff/reader.go b/src/pkg/image/tiff/reader.go new file mode 100644 index 000000000..26e52144d --- /dev/null +++ b/src/pkg/image/tiff/reader.go @@ -0,0 +1,419 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tiff implements a TIFF image decoder. +// +// The TIFF specification is at http://partners.adobe.com/public/developer/en/tiff/TIFF6.pdf +package tiff + +import ( + "compress/lzw" + "compress/zlib" + "encoding/binary" + "image" + "io" + "io/ioutil" + "os" +) + +// A FormatError reports that the input is not a valid TIFF image. +type FormatError string + +func (e FormatError) String() string { + return "tiff: invalid format: " + string(e) +} + +// An UnsupportedError reports that the input uses a valid but +// unimplemented feature. +type UnsupportedError string + +func (e UnsupportedError) String() string { + return "tiff: unsupported feature: " + string(e) +} + +// An InternalError reports that an internal error was encountered. +type InternalError string + +func (e InternalError) String() string { + return "tiff: internal error: " + string(e) +} + +type decoder struct { + r io.ReaderAt + byteOrder binary.ByteOrder + config image.Config + mode imageMode + features map[int][]uint + palette []image.Color + + buf []byte + off int // Current offset in buf. + v uint32 // Buffer value for reading with arbitrary bit depths. + nbits uint // Remaining number of bits in v. +} + +// firstVal returns the first uint of the features entry with the given tag, +// or 0 if the tag does not exist. +func (d *decoder) firstVal(tag int) uint { + f := d.features[tag] + if len(f) == 0 { + return 0 + } + return f[0] +} + +// ifdUint decodes the IFD entry in p, which must be of the Byte, Short +// or Long type, and returns the decoded uint values. +func (d *decoder) ifdUint(p []byte) (u []uint, err os.Error) { + var raw []byte + datatype := d.byteOrder.Uint16(p[2:4]) + count := d.byteOrder.Uint32(p[4:8]) + if datalen := lengths[datatype] * count; datalen > 4 { + // The IFD contains a pointer to the real value. + raw = make([]byte, datalen) + _, err = d.r.ReadAt(raw, int64(d.byteOrder.Uint32(p[8:12]))) + } else { + raw = p[8 : 8+datalen] + } + if err != nil { + return nil, err + } + + u = make([]uint, count) + switch datatype { + case dtByte: + for i := uint32(0); i < count; i++ { + u[i] = uint(raw[i]) + } + case dtShort: + for i := uint32(0); i < count; i++ { + u[i] = uint(d.byteOrder.Uint16(raw[2*i : 2*(i+1)])) + } + case dtLong: + for i := uint32(0); i < count; i++ { + u[i] = uint(d.byteOrder.Uint32(raw[4*i : 4*(i+1)])) + } + default: + return nil, UnsupportedError("data type") + } + return u, nil +} + +// parseIFD decides whether the the IFD entry in p is "interesting" and +// stows away the data in the decoder. +func (d *decoder) parseIFD(p []byte) os.Error { + tag := d.byteOrder.Uint16(p[0:2]) + switch tag { + case tBitsPerSample, + tExtraSamples, + tPhotometricInterpretation, + tCompression, + tPredictor, + tStripOffsets, + tStripByteCounts, + tRowsPerStrip, + tImageLength, + tImageWidth: + val, err := d.ifdUint(p) + if err != nil { + return err + } + d.features[int(tag)] = val + case tColorMap: + val, err := d.ifdUint(p) + if err != nil { + return err + } + numcolors := len(val) / 3 + if len(val)%3 != 0 || numcolors <= 0 || numcolors > 256 { + return FormatError("bad ColorMap length") + } + d.palette = make([]image.Color, numcolors) + for i := 0; i < numcolors; i++ { + d.palette[i] = image.RGBA64Color{ + uint16(val[i]), + uint16(val[i+numcolors]), + uint16(val[i+2*numcolors]), + 0xffff, + } + } + case tSampleFormat: + // Page 27 of the spec: If the SampleFormat is present and + // the value is not 1 [= unsigned integer data], a Baseline + // TIFF reader that cannot handle the SampleFormat value + // must terminate the import process gracefully. + val, err := d.ifdUint(p) + if err != nil { + return err + } + for _, v := range val { + if v != 1 { + return UnsupportedError("sample format") + } + } + } + return nil +} + +// readBits reads n bits from the internal buffer starting at the current offset. +func (d *decoder) readBits(n uint) uint32 { + for d.nbits < n { + d.v <<= 8 + d.v |= uint32(d.buf[d.off]) + d.off++ + d.nbits += 8 + } + d.nbits -= n + rv := d.v >> d.nbits + d.v &^= rv << d.nbits + return rv +} + +// flushBits discards the unread bits in the buffer used by readBits. +// It is used at the end of a line. +func (d *decoder) flushBits() { + d.v = 0 + d.nbits = 0 +} + +// decode decodes the raw data of an image. +// It reads from d.buf and writes the strip with ymin <= y < ymax into dst. +func (d *decoder) decode(dst image.Image, ymin, ymax int) os.Error { + spp := len(d.features[tBitsPerSample]) // samples per pixel + d.off = 0 + width := dst.Bounds().Dx() + + // Apply horizontal predictor if necessary. + // In this case, p contains the color difference to the preceding pixel. + // See page 64-65 of the spec. + if d.firstVal(tPredictor) == prHorizontal && d.firstVal(tBitsPerSample) == 8 { + for y := ymin; y < ymax; y++ { + d.off += spp + for x := 0; x < (width-1)*spp; x++ { + d.buf[d.off] += d.buf[d.off-spp] + d.off++ + } + } + d.off = 0 + } + + switch d.mode { + case mGray, mGrayInvert: + img := dst.(*image.Gray) + bpp := d.firstVal(tBitsPerSample) + max := uint32((1 << bpp) - 1) + for y := ymin; y < ymax; y++ { + for x := img.Rect.Min.X; x < img.Rect.Max.X; x++ { + v := uint8(d.readBits(bpp) * 0xff / max) + if d.mode == mGrayInvert { + v = 0xff - v + } + img.SetGray(x, y, image.GrayColor{v}) + } + d.flushBits() + } + case mPaletted: + img := dst.(*image.Paletted) + bpp := d.firstVal(tBitsPerSample) + for y := ymin; y < ymax; y++ { + for x := img.Rect.Min.X; x < img.Rect.Max.X; x++ { + img.SetColorIndex(x, y, uint8(d.readBits(bpp))) + } + d.flushBits() + } + case mRGB: + img := dst.(*image.RGBA) + for y := ymin; y < ymax; y++ { + for x := img.Rect.Min.X; x < img.Rect.Max.X; x++ { + img.SetRGBA(x, y, image.RGBAColor{d.buf[d.off], d.buf[d.off+1], d.buf[d.off+2], 0xff}) + d.off += spp + } + } + case mNRGBA: + img := dst.(*image.NRGBA) + for y := ymin; y < ymax; y++ { + for x := img.Rect.Min.X; x < img.Rect.Max.X; x++ { + img.SetNRGBA(x, y, image.NRGBAColor{d.buf[d.off], d.buf[d.off+1], d.buf[d.off+2], d.buf[d.off+3]}) + d.off += spp + } + } + case mRGBA: + img := dst.(*image.RGBA) + for y := ymin; y < ymax; y++ { + for x := img.Rect.Min.X; x < img.Rect.Max.X; x++ { + img.SetRGBA(x, y, image.RGBAColor{d.buf[d.off], d.buf[d.off+1], d.buf[d.off+2], d.buf[d.off+3]}) + d.off += spp + } + } + } + + return nil +} + +func newDecoder(r io.Reader) (*decoder, os.Error) { + d := &decoder{ + r: newReaderAt(r), + features: make(map[int][]uint), + } + + p := make([]byte, 8) + if _, err := d.r.ReadAt(p, 0); err != nil { + return nil, err + } + switch string(p[0:4]) { + case leHeader: + d.byteOrder = binary.LittleEndian + case beHeader: + d.byteOrder = binary.BigEndian + default: + return nil, FormatError("malformed header") + } + + ifdOffset := int64(d.byteOrder.Uint32(p[4:8])) + + // The first two bytes contain the number of entries (12 bytes each). + if _, err := d.r.ReadAt(p[0:2], ifdOffset); err != nil { + return nil, err + } + numItems := int(d.byteOrder.Uint16(p[0:2])) + + // All IFD entries are read in one chunk. + p = make([]byte, ifdLen*numItems) + if _, err := d.r.ReadAt(p, ifdOffset+2); err != nil { + return nil, err + } + + for i := 0; i < len(p); i += ifdLen { + if err := d.parseIFD(p[i : i+ifdLen]); err != nil { + return nil, err + } + } + + d.config.Width = int(d.firstVal(tImageWidth)) + d.config.Height = int(d.firstVal(tImageLength)) + + if _, ok := d.features[tBitsPerSample]; !ok { + return nil, FormatError("BitsPerSample tag missing") + } + + // Determine the image mode. + switch d.firstVal(tPhotometricInterpretation) { + case pRGB: + for _, b := range d.features[tBitsPerSample] { + if b != 8 { + return nil, UnsupportedError("non-8-bit RGB image") + } + } + d.config.ColorModel = image.RGBAColorModel + // RGB images normally have 3 samples per pixel. + // If there are more, ExtraSamples (p. 31-32 of the spec) + // gives their meaning (usually an alpha channel). + switch len(d.features[tBitsPerSample]) { + case 3: + d.mode = mRGB + case 4: + switch d.firstVal(tExtraSamples) { + case 1: + d.mode = mRGBA + case 2: + d.mode = mNRGBA + d.config.ColorModel = image.NRGBAColorModel + default: + // The extra sample is discarded. + d.mode = mRGB + } + default: + return nil, FormatError("wrong number of samples for RGB") + } + case pPaletted: + d.mode = mPaletted + d.config.ColorModel = image.PalettedColorModel(d.palette) + case pWhiteIsZero: + d.mode = mGrayInvert + d.config.ColorModel = image.GrayColorModel + case pBlackIsZero: + d.mode = mGray + d.config.ColorModel = image.GrayColorModel + default: + return nil, UnsupportedError("color model") + } + + return d, nil +} + +// DecodeConfig returns the color model and dimensions of a TIFF image without +// decoding the entire image. +func DecodeConfig(r io.Reader) (image.Config, os.Error) { + d, err := newDecoder(r) + if err != nil { + return image.Config{}, err + } + return d.config, nil +} + +// Decode reads a TIFF image from r and returns it as an image.Image. +// The type of Image returned depends on the contents of the TIFF. +func Decode(r io.Reader) (img image.Image, err os.Error) { + d, err := newDecoder(r) + if err != nil { + return + } + + // Check if we have the right number of strips, offsets and counts. + rps := int(d.firstVal(tRowsPerStrip)) + numStrips := (d.config.Height + rps - 1) / rps + if rps == 0 || len(d.features[tStripOffsets]) < numStrips || len(d.features[tStripByteCounts]) < numStrips { + return nil, FormatError("inconsistent header") + } + + switch d.mode { + case mGray, mGrayInvert: + img = image.NewGray(d.config.Width, d.config.Height) + case mPaletted: + img = image.NewPaletted(d.config.Width, d.config.Height, d.palette) + case mNRGBA: + img = image.NewNRGBA(d.config.Width, d.config.Height) + case mRGB, mRGBA: + img = image.NewRGBA(d.config.Width, d.config.Height) + } + + for i := 0; i < numStrips; i++ { + ymin := i * rps + // The last strip may be shorter. + if i == numStrips-1 && d.config.Height%rps != 0 { + rps = d.config.Height % rps + } + offset := int64(d.features[tStripOffsets][i]) + n := int64(d.features[tStripByteCounts][i]) + switch d.firstVal(tCompression) { + case cNone: + // TODO(bsiegert): Avoid copy if r is a tiff.buffer. + d.buf = make([]byte, n) + _, err = d.r.ReadAt(d.buf, offset) + case cLZW: + r := lzw.NewReader(io.NewSectionReader(d.r, offset, n), lzw.MSB, 8) + d.buf, err = ioutil.ReadAll(r) + r.Close() + case cDeflate, cDeflateOld: + r, err := zlib.NewReader(io.NewSectionReader(d.r, offset, n)) + if err != nil { + return nil, err + } + d.buf, err = ioutil.ReadAll(r) + r.Close() + default: + err = UnsupportedError("compression") + } + if err != nil { + return + } + err = d.decode(img, ymin, ymin+rps) + } + return +} + +func init() { + image.RegisterFormat("tiff", leHeader, Decode, DecodeConfig) + image.RegisterFormat("tiff", beHeader, Decode, DecodeConfig) +} diff --git a/src/pkg/image/ycbcr/ycbcr.go b/src/pkg/image/ycbcr/ycbcr.go index cda45996d..f2de3d6fb 100644 --- a/src/pkg/image/ycbcr/ycbcr.go +++ b/src/pkg/image/ycbcr/ycbcr.go @@ -142,7 +142,7 @@ func (p *YCbCr) Bounds() image.Rectangle { } func (p *YCbCr) At(x, y int) image.Color { - if !p.Rect.Contains(image.Point{x, y}) { + if !(image.Point{x, y}.In(p.Rect)) { return YCbCrColor{} } switch p.SubsampleRatio { @@ -169,6 +169,15 @@ func (p *YCbCr) At(x, y int) image.Color { } } +// SubImage returns an image representing the portion of the image p visible +// through r. The returned value shares pixels with the original image. +func (p *YCbCr) SubImage(r image.Rectangle) image.Image { + q := new(YCbCr) + *q = *p + q.Rect = q.Rect.Intersect(r) + return q +} + func (p *YCbCr) Opaque() bool { return true } diff --git a/src/pkg/index/suffixarray/suffixarray_test.go b/src/pkg/index/suffixarray/suffixarray_test.go index e85267f17..b1499027a 100644 --- a/src/pkg/index/suffixarray/suffixarray_test.go +++ b/src/pkg/index/suffixarray/suffixarray_test.go @@ -108,7 +108,7 @@ var testCases = []testCase{ } -// find all occurrences of s in source; report at most n occurences +// find all occurrences of s in source; report at most n occurrences func find(src, s string, n int) []int { var res vector.IntVector if s != "" && n != 0 { diff --git a/src/pkg/io/io.go b/src/pkg/io/io.go index 0bc73d67d..846dcacb5 100644 --- a/src/pkg/io/io.go +++ b/src/pkg/io/io.go @@ -162,6 +162,18 @@ type ByteReader interface { ReadByte() (c byte, err os.Error) } +// ByteScanner is the interface that adds the UnreadByte method to the +// basic ReadByte method. +// +// UnreadByte causes the next call to ReadByte to return the same byte +// as the previous call to ReadByte. +// It may be an error to call UnreadByte twice without an intervening +// call to ReadByte. +type ByteScanner interface { + ByteReader + UnreadByte() os.Error +} + // RuneReader is the interface that wraps the ReadRune method. // // ReadRune reads a single UTF-8 encoded Unicode character @@ -171,6 +183,18 @@ type RuneReader interface { ReadRune() (rune int, size int, err os.Error) } +// RuneScanner is the interface that adds the UnreadRune method to the +// basic ReadRune method. +// +// UnreadRune causes the next call to ReadRune to return the same rune +// as the previous call to ReadRune. +// It may be an error to call UnreadRune twice without an intervening +// call to ReadRune. +type RuneScanner interface { + RuneReader + UnreadRune() os.Error +} + // WriteString writes the contents of the string s to w, which accepts an array of bytes. func WriteString(w Writer, s string) (n int, err os.Error) { return w.Write([]byte(s)) @@ -303,22 +327,26 @@ func Copy(dst Writer, src Reader) (written int64, err os.Error) { // LimitReader returns a Reader that reads from r // but stops with os.EOF after n bytes. -func LimitReader(r Reader, n int64) Reader { return &limitedReader{r, n} } - -type limitedReader struct { - r Reader - n int64 +// The underlying implementation is a *LimitedReader. +func LimitReader(r Reader, n int64) Reader { return &LimitedReader{r, n} } + +// A LimitedReader reads from R but limits the amount of +// data returned to just N bytes. Each call to Read +// updates N to reflect the new amount remaining. +type LimitedReader struct { + R Reader // underlying reader + N int64 // max bytes remaining } -func (l *limitedReader) Read(p []byte) (n int, err os.Error) { - if l.n <= 0 { +func (l *LimitedReader) Read(p []byte) (n int, err os.Error) { + if l.N <= 0 { return 0, os.EOF } - if int64(len(p)) > l.n { - p = p[0:l.n] + if int64(len(p)) > l.N { + p = p[0:l.N] } - n, err = l.r.Read(p) - l.n -= int64(n) + n, err = l.R.Read(p) + l.N -= int64(n) return } diff --git a/src/pkg/io/multi_test.go b/src/pkg/io/multi_test.go index 3ecb7c75d..1b3589dde 100644 --- a/src/pkg/io/multi_test.go +++ b/src/pkg/io/multi_test.go @@ -20,8 +20,9 @@ func TestMultiReader(t *testing.T) { nread := 0 withFooBar := func(tests func()) { r1 := strings.NewReader("foo ") - r2 := strings.NewReader("bar") - mr = MultiReader(r1, r2) + r2 := strings.NewReader("") + r3 := strings.NewReader("bar") + mr = MultiReader(r1, r2, r3) buf = make([]byte, 20) tests() } diff --git a/src/pkg/mail/Makefile b/src/pkg/mail/Makefile new file mode 100644 index 000000000..e4de5428e --- /dev/null +++ b/src/pkg/mail/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=mail +GOFILES=\ + message.go\ + +include ../../Make.pkg diff --git a/src/pkg/mail/message.go b/src/pkg/mail/message.go new file mode 100644 index 000000000..754b779be --- /dev/null +++ b/src/pkg/mail/message.go @@ -0,0 +1,491 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package mail implements parsing of mail messages. + +For the most part, this package follows the syntax as specified by RFC 5322. +Notable divergences: + * Obsolete address formats are not parsed, including addresses with + embedded route information. + * Group addresses are not parsed. + * The full range of spacing (the CFWS syntax element) is not supported, + such as breaking addresses across lines. +*/ +package mail + +import ( + "bufio" + "bytes" + "fmt" + "io" + "log" + "net/textproto" + "os" + "strconv" + "strings" + "time" +) + +var debug = debugT(false) + +type debugT bool + +func (d debugT) Printf(format string, args ...interface{}) { + if d { + log.Printf(format, args...) + } +} + +// A Message represents a parsed mail message. +type Message struct { + Header Header + Body io.Reader +} + +// ReadMessage reads a message from r. +// The headers are parsed, and the body of the message will be reading from r. +func ReadMessage(r io.Reader) (msg *Message, err os.Error) { + tp := textproto.NewReader(bufio.NewReader(r)) + + hdr, err := tp.ReadMIMEHeader() + if err != nil { + return nil, err + } + + return &Message{ + Header: Header(hdr), + Body: tp.R, + }, nil +} + +// Layouts suitable for passing to time.Parse. +// These are tried in order. +var dateLayouts []string + +func init() { + // Generate layouts based on RFC 5322, section 3.3. + + dows := [...]string{"", "Mon, "} // day-of-week + days := [...]string{"2", "02"} // day = 1*2DIGIT + years := [...]string{"2006", "06"} // year = 4*DIGIT / 2*DIGIT + seconds := [...]string{":05", ""} // second + zones := [...]string{"-0700", "MST"} // zone = (("+" / "-") 4DIGIT) / "GMT" / ... + + for _, dow := range dows { + for _, day := range days { + for _, year := range years { + for _, second := range seconds { + for _, zone := range zones { + s := dow + day + " Jan " + year + " 15:04" + second + " " + zone + dateLayouts = append(dateLayouts, s) + } + } + } + } + } +} + +func parseDate(date string) (*time.Time, os.Error) { + for _, layout := range dateLayouts { + t, err := time.Parse(layout, date) + if err == nil { + return t, nil + } + } + return nil, os.ErrorString("mail: header could not be parsed") +} + +// A Header represents the key-value pairs in a mail message header. +type Header map[string][]string + +// Get gets the first value associated with the given key. +// If there are no values associated with the key, Get returns "". +func (h Header) Get(key string) string { + return textproto.MIMEHeader(h).Get(key) +} + +var ErrHeaderNotPresent = os.ErrorString("mail: header not in message") + +// Date parses the Date header field. +func (h Header) Date() (*time.Time, os.Error) { + hdr := h.Get("Date") + if hdr == "" { + return nil, ErrHeaderNotPresent + } + return parseDate(hdr) +} + +// AddressList parses the named header field as a list of addresses. +func (h Header) AddressList(key string) ([]*Address, os.Error) { + hdr := h.Get(key) + if hdr == "" { + return nil, ErrHeaderNotPresent + } + return newAddrParser(hdr).parseAddressList() +} + +// Address represents a single mail address. +// An address such as "Barry Gibbs <bg@example.com>" is represented +// as Address{Name: "Barry Gibbs", Address: "bg@example.com"}. +type Address struct { + Name string // Proper name; may be empty. + Address string // user@domain +} + +// String formats the address as a valid RFC 5322 address. +// If the address's name contains non-ASCII characters +// the name will be rendered according to RFC 2047. +func (a *Address) String() string { + s := "<" + a.Address + ">" + if a.Name == "" { + return s + } + // If every character is printable ASCII, quoting is simple. + allPrintable := true + for i := 0; i < len(a.Name); i++ { + if !isVchar(a.Name[i]) { + allPrintable = false + break + } + } + if allPrintable { + b := bytes.NewBufferString(`"`) + for i := 0; i < len(a.Name); i++ { + if !isQtext(a.Name[i]) { + b.WriteByte('\\') + } + b.WriteByte(a.Name[i]) + } + b.WriteString(`" `) + b.WriteString(s) + return b.String() + } + + // UTF-8 "Q" encoding + b := bytes.NewBufferString("=?utf-8?q?") + for i := 0; i < len(a.Name); i++ { + switch c := a.Name[i]; { + case c == ' ': + b.WriteByte('_') + case isVchar(c) && c != '=' && c != '?' && c != '_': + b.WriteByte(c) + default: + fmt.Fprintf(b, "=%02X", c) + } + } + b.WriteString("?= ") + b.WriteString(s) + return b.String() +} + +type addrParser []byte + +func newAddrParser(s string) *addrParser { + p := addrParser([]byte(s)) + return &p +} + +func (p *addrParser) parseAddressList() ([]*Address, os.Error) { + var list []*Address + for { + p.skipSpace() + addr, err := p.parseAddress() + if err != nil { + return nil, err + } + list = append(list, addr) + + p.skipSpace() + if p.empty() { + break + } + if !p.consume(',') { + return nil, os.ErrorString("mail: expected comma") + } + } + return list, nil +} + +// parseAddress parses a single RFC 5322 address at the start of p. +func (p *addrParser) parseAddress() (addr *Address, err os.Error) { + debug.Printf("parseAddress: %q", *p) + p.skipSpace() + if p.empty() { + return nil, os.ErrorString("mail: no address") + } + + // address = name-addr / addr-spec + // TODO(dsymonds): Support parsing group address. + + // addr-spec has a more restricted grammar than name-addr, + // so try parsing it first, and fallback to name-addr. + // TODO(dsymonds): Is this really correct? + spec, err := p.consumeAddrSpec() + if err == nil { + return &Address{ + Address: spec, + }, err + } + debug.Printf("parseAddress: not an addr-spec: %v", err) + debug.Printf("parseAddress: state is now %q", *p) + + // display-name + var displayName string + if p.peek() != '<' { + displayName, err = p.consumePhrase() + if err != nil { + return nil, err + } + } + debug.Printf("parseAddress: displayName=%q", displayName) + + // angle-addr = "<" addr-spec ">" + p.skipSpace() + if !p.consume('<') { + return nil, os.ErrorString("mail: no angle-addr") + } + spec, err = p.consumeAddrSpec() + if err != nil { + return nil, err + } + if !p.consume('>') { + return nil, os.ErrorString("mail: unclosed angle-addr") + } + debug.Printf("parseAddress: spec=%q", spec) + + return &Address{ + Name: displayName, + Address: spec, + }, nil +} + +// consumeAddrSpec parses a single RFC 5322 addr-spec at the start of p. +func (p *addrParser) consumeAddrSpec() (spec string, err os.Error) { + debug.Printf("consumeAddrSpec: %q", *p) + + orig := *p + defer func() { + if err != nil { + *p = orig + } + }() + + // local-part = dot-atom / quoted-string + var localPart string + p.skipSpace() + if p.empty() { + return "", os.ErrorString("mail: no addr-spec") + } + if p.peek() == '"' { + // quoted-string + debug.Printf("consumeAddrSpec: parsing quoted-string") + localPart, err = p.consumeQuotedString() + } else { + // dot-atom + debug.Printf("consumeAddrSpec: parsing dot-atom") + localPart, err = p.consumeAtom(true) + } + if err != nil { + debug.Printf("consumeAddrSpec: failed: %v", err) + return "", err + } + + if !p.consume('@') { + return "", os.ErrorString("mail: missing @ in addr-spec") + } + + // domain = dot-atom / domain-literal + var domain string + p.skipSpace() + if p.empty() { + return "", os.ErrorString("mail: no domain in addr-spec") + } + // TODO(dsymonds): Handle domain-literal + domain, err = p.consumeAtom(true) + if err != nil { + return "", err + } + + return localPart + "@" + domain, nil +} + +// consumePhrase parses the RFC 5322 phrase at the start of p. +func (p *addrParser) consumePhrase() (phrase string, err os.Error) { + debug.Printf("consumePhrase: [%s]", *p) + // phrase = 1*word + var words []string + for { + // word = atom / quoted-string + var word string + p.skipSpace() + if p.empty() { + return "", os.ErrorString("mail: missing phrase") + } + if p.peek() == '"' { + // quoted-string + word, err = p.consumeQuotedString() + } else { + // atom + word, err = p.consumeAtom(false) + } + + // RFC 2047 encoded-word starts with =?, ends with ?=, and has two other ?s. + if err == nil && strings.HasPrefix(word, "=?") && strings.HasSuffix(word, "?=") && strings.Count(word, "?") == 4 { + word, err = decodeRFC2047Word(word) + } + + if err != nil { + break + } + debug.Printf("consumePhrase: consumed %q", word) + words = append(words, word) + } + // Ignore any error if we got at least one word. + if err != nil && len(words) == 0 { + debug.Printf("consumePhrase: hit err: %v", err) + return "", os.ErrorString("mail: missing word in phrase") + } + phrase = strings.Join(words, " ") + return phrase, nil +} + +// consumeQuotedString parses the quoted string at the start of p. +func (p *addrParser) consumeQuotedString() (qs string, err os.Error) { + // Assume first byte is '"'. + i := 1 + qsb := make([]byte, 0, 10) +Loop: + for { + if i >= p.len() { + return "", os.ErrorString("mail: unclosed quoted-string") + } + switch c := (*p)[i]; { + case c == '"': + break Loop + case c == '\\': + if i+1 == p.len() { + return "", os.ErrorString("mail: unclosed quoted-string") + } + qsb = append(qsb, (*p)[i+1]) + i += 2 + case isQtext(c), c == ' ' || c == '\t': + // qtext (printable US-ASCII excluding " and \), or + // FWS (almost; we're ignoring CRLF) + qsb = append(qsb, c) + i++ + default: + return "", fmt.Errorf("mail: bad character in quoted-string: %q", c) + } + } + *p = (*p)[i+1:] + return string(qsb), nil +} + +// consumeAtom parses an RFC 5322 atom at the start of p. +// If dot is true, consumeAtom parses an RFC 5322 dot-atom instead. +func (p *addrParser) consumeAtom(dot bool) (atom string, err os.Error) { + if !isAtext(p.peek(), false) { + return "", os.ErrorString("mail: invalid string") + } + i := 1 + for ; i < p.len() && isAtext((*p)[i], dot); i++ { + } + // TODO(dsymonds): Remove the []byte() conversion here when 6g doesn't need it. + atom, *p = string([]byte((*p)[:i])), (*p)[i:] + return atom, nil +} + +func (p *addrParser) consume(c byte) bool { + if p.empty() || p.peek() != c { + return false + } + *p = (*p)[1:] + return true +} + +// skipSpace skips the leading space and tab characters. +func (p *addrParser) skipSpace() { + *p = bytes.TrimLeft(*p, " \t") +} + +func (p *addrParser) peek() byte { + return (*p)[0] +} + +func (p *addrParser) empty() bool { + return p.len() == 0 +} + +func (p *addrParser) len() int { + return len(*p) +} + +func decodeRFC2047Word(s string) (string, os.Error) { + fields := strings.Split(s, "?", -1) + if len(fields) != 5 || fields[0] != "=" || fields[4] != "=" { + return "", os.ErrorString("mail: address not RFC 2047 encoded") + } + charset, enc := strings.ToLower(fields[1]), strings.ToLower(fields[2]) + // TODO(dsymonds): Support "b" encoding too. + if enc != "q" { + return "", fmt.Errorf("mail: RFC 2047 encoding not supported: %q", enc) + } + if charset != "iso-8859-1" && charset != "utf-8" { + return "", fmt.Errorf("mail: charset not supported: %q", charset) + } + + in := fields[3] + b := new(bytes.Buffer) + for i := 0; i < len(in); i++ { + switch c := in[i]; { + case c == '=' && i+2 < len(in): + x, err := strconv.Btoi64(in[i+1:i+3], 16) + if err != nil { + return "", fmt.Errorf("mail: invalid RFC 2047 encoding: %q", in[i:i+3]) + } + i += 2 + switch charset { + case "iso-8859-1": + b.WriteRune(int(x)) + case "utf-8": + b.WriteByte(byte(x)) + } + case c == '_': + b.WriteByte(' ') + default: + b.WriteByte(c) + } + } + return b.String(), nil +} + +var atextChars = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + "abcdefghijklmnopqrstuvwxyz" + + "0123456789" + + "!#$%&'*+-/=?^_`{|}~") + +// isAtext returns true if c is an RFC 5322 atext character. +// If dot is true, period is included. +func isAtext(c byte, dot bool) bool { + if dot && c == '.' { + return true + } + return bytes.IndexByte(atextChars, c) >= 0 +} + +// isQtext returns true if c is an RFC 5322 qtest character. +func isQtext(c byte) bool { + // Printable US-ASCII, excluding backslash or quote. + if c == '\\' || c == '"' { + return false + } + return '!' <= c && c <= '~' +} + +// isVchar returns true if c is an RFC 5322 VCHAR character. +func isVchar(c byte) bool { + // Visible (printing) characters. + return '!' <= c && c <= '~' +} diff --git a/src/pkg/mail/message_test.go b/src/pkg/mail/message_test.go new file mode 100644 index 000000000..1ff45d2c1 --- /dev/null +++ b/src/pkg/mail/message_test.go @@ -0,0 +1,258 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mail + +import ( + "bytes" + "io/ioutil" + "reflect" + "testing" + "time" +) + +var parseTests = []struct { + in string + header Header + body string +}{ + { + // RFC 5322, Appendix A.1.1 + in: `From: John Doe <jdoe@machine.example> +To: Mary Smith <mary@example.net> +Subject: Saying Hello +Date: Fri, 21 Nov 1997 09:55:06 -0600 +Message-ID: <1234@local.machine.example> + +This is a message just to say hello. +So, "Hello". +`, + header: Header{ + "From": []string{"John Doe <jdoe@machine.example>"}, + "To": []string{"Mary Smith <mary@example.net>"}, + "Subject": []string{"Saying Hello"}, + "Date": []string{"Fri, 21 Nov 1997 09:55:06 -0600"}, + "Message-Id": []string{"<1234@local.machine.example>"}, + }, + body: "This is a message just to say hello.\nSo, \"Hello\".\n", + }, +} + +func TestParsing(t *testing.T) { + for i, test := range parseTests { + msg, err := ReadMessage(bytes.NewBuffer([]byte(test.in))) + if err != nil { + t.Errorf("test #%d: Failed parsing message: %v", i, err) + continue + } + if !headerEq(msg.Header, test.header) { + t.Errorf("test #%d: Incorrectly parsed message header.\nGot:\n%+v\nWant:\n%+v", + i, msg.Header, test.header) + } + body, err := ioutil.ReadAll(msg.Body) + if err != nil { + t.Errorf("test #%d: Failed reading body: %v", i, err) + continue + } + bodyStr := string(body) + if bodyStr != test.body { + t.Errorf("test #%d: Incorrectly parsed message body.\nGot:\n%+v\nWant:\n%+v", + i, bodyStr, test.body) + } + } +} + +func headerEq(a, b Header) bool { + if len(a) != len(b) { + return false + } + for k, as := range a { + bs, ok := b[k] + if !ok { + return false + } + if !reflect.DeepEqual(as, bs) { + return false + } + } + return true +} + +func TestDateParsing(t *testing.T) { + tests := []struct { + dateStr string + exp *time.Time + }{ + // RFC 5322, Appendix A.1.1 + { + "Fri, 21 Nov 1997 09:55:06 -0600", + &time.Time{ + Year: 1997, + Month: 11, + Day: 21, + Hour: 9, + Minute: 55, + Second: 6, + Weekday: 5, // Fri + ZoneOffset: -6 * 60 * 60, + }, + }, + // RFC5322, Appendix A.6.2 + // Obsolete date. + { + "21 Nov 97 09:55:06 GMT", + &time.Time{ + Year: 1997, + Month: 11, + Day: 21, + Hour: 9, + Minute: 55, + Second: 6, + Zone: "GMT", + }, + }, + } + for _, test := range tests { + hdr := Header{ + "Date": []string{test.dateStr}, + } + date, err := hdr.Date() + if err != nil { + t.Errorf("Failed parsing %q: %v", test.dateStr, err) + continue + } + if !reflect.DeepEqual(date, test.exp) { + t.Errorf("Parse of %q: got %+v, want %+v", test.dateStr, date, test.exp) + } + } +} + +func TestAddressParsing(t *testing.T) { + tests := []struct { + addrsStr string + exp []*Address + }{ + // Bare address + { + `jdoe@machine.example`, + []*Address{&Address{ + Address: "jdoe@machine.example", + }}, + }, + // RFC 5322, Appendix A.1.1 + { + `John Doe <jdoe@machine.example>`, + []*Address{&Address{ + Name: "John Doe", + Address: "jdoe@machine.example", + }}, + }, + // RFC 5322, Appendix A.1.2 + { + `"Joe Q. Public" <john.q.public@example.com>`, + []*Address{&Address{ + Name: "Joe Q. Public", + Address: "john.q.public@example.com", + }}, + }, + { + `Mary Smith <mary@x.test>, jdoe@example.org, Who? <one@y.test>`, + []*Address{ + &Address{ + Name: "Mary Smith", + Address: "mary@x.test", + }, + &Address{ + Address: "jdoe@example.org", + }, + &Address{ + Name: "Who?", + Address: "one@y.test", + }, + }, + }, + { + `<boss@nil.test>, "Giant; \"Big\" Box" <sysservices@example.net>`, + []*Address{ + &Address{ + Address: "boss@nil.test", + }, + &Address{ + Name: `Giant; "Big" Box`, + Address: "sysservices@example.net", + }, + }, + }, + // RFC 5322, Appendix A.1.3 + // TODO(dsymonds): Group addresses. + + // RFC 2047 "Q"-encoded ISO-8859-1 address. + { + `=?iso-8859-1?q?J=F6rg_Doe?= <joerg@example.com>`, + []*Address{ + &Address{ + Name: `Jörg Doe`, + Address: "joerg@example.com", + }, + }, + }, + // RFC 2047 "Q"-encoded UTF-8 address. + { + `=?utf-8?q?J=C3=B6rg_Doe?= <joerg@example.com>`, + []*Address{ + &Address{ + Name: `Jörg Doe`, + Address: "joerg@example.com", + }, + }, + }, + // RFC 2047, Section 8. + { + `=?ISO-8859-1?Q?Andr=E9?= Pirard <PIRARD@vm1.ulg.ac.be>`, + []*Address{ + &Address{ + Name: `André Pirard`, + Address: "PIRARD@vm1.ulg.ac.be", + }, + }, + }, + } + for _, test := range tests { + addrs, err := newAddrParser(test.addrsStr).parseAddressList() + if err != nil { + t.Errorf("Failed parsing %q: %v", test.addrsStr, err) + continue + } + if !reflect.DeepEqual(addrs, test.exp) { + t.Errorf("Parse of %q: got %+v, want %+v", test.addrsStr, addrs, test.exp) + } + } +} + +func TestAddressFormatting(t *testing.T) { + tests := []struct { + addr *Address + exp string + }{ + { + &Address{Address: "bob@example.com"}, + "<bob@example.com>", + }, + { + &Address{Name: "Bob", Address: "bob@example.com"}, + `"Bob" <bob@example.com>`, + }, + { + // note the ö (o with an umlaut) + &Address{Name: "Böb", Address: "bob@example.com"}, + `=?utf-8?q?B=C3=B6b?= <bob@example.com>`, + }, + } + for _, test := range tests { + s := test.addr.String() + if s != test.exp { + t.Errorf("Address%+v.String() = %v, want %v", *test.addr, s, test.exp) + } + } +} diff --git a/src/pkg/math/Makefile b/src/pkg/math/Makefile index 71347b7fa..8e8e74ae4 100644 --- a/src/pkg/math/Makefile +++ b/src/pkg/math/Makefile @@ -6,6 +6,9 @@ include ../../Make.inc TARG=math +OFILES_arm=\ + sqrt_arm.$O\ + OFILES_amd64=\ exp_amd64.$O\ fabs_amd64.$O\ diff --git a/src/pkg/math/log.go b/src/pkg/math/log.go index 39d94512d..a786c8ce3 100644 --- a/src/pkg/math/log.go +++ b/src/pkg/math/log.go @@ -23,7 +23,7 @@ package math // ==================================================== // // __ieee754_log(x) -// Return the logrithm of x +// Return the logarithm of x // // Method : // 1. Argument Reduction: find k and f such that diff --git a/src/pkg/math/sqrt_arm.s b/src/pkg/math/sqrt_arm.s new file mode 100644 index 000000000..befbb8a89 --- /dev/null +++ b/src/pkg/math/sqrt_arm.s @@ -0,0 +1,10 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// func Sqrt(x float64) float64 +TEXT ·Sqrt(SB),7,$0 + MOVD x+0(FP),F0 + SQRTD F0,F0 + MOVD F0,r+8(FP) + RET diff --git a/src/pkg/math/sqrt_port.go b/src/pkg/math/sqrt_port.go index 6f35a383d..148239bcf 100644 --- a/src/pkg/math/sqrt_port.go +++ b/src/pkg/math/sqrt_port.go @@ -50,7 +50,7 @@ package math // If (2) is false, then q = q ; otherwise q = q + 2 . // i+1 i i+1 i // -// With some algebric manipulation, it is not difficult to see +// With some algebraic manipulation, it is not difficult to see // that (2) is equivalent to // -(i+1) // s + 2 <= y (3) @@ -141,3 +141,7 @@ func sqrtGo(x float64) float64 { ix = q>>1 + uint64(exp-1+bias)<<shift // significand + biased exponent return Float64frombits(ix) } + +func sqrtGoC(f float64, r *float64) { + *r = sqrtGo(f) +} diff --git a/src/pkg/mime/multipart/Makefile b/src/pkg/mime/multipart/Makefile index 5051f0df1..de1a439f2 100644 --- a/src/pkg/mime/multipart/Makefile +++ b/src/pkg/mime/multipart/Makefile @@ -8,5 +8,6 @@ TARG=mime/multipart GOFILES=\ formdata.go\ multipart.go\ + writer.go\ include ../../../Make.pkg diff --git a/src/pkg/mime/multipart/formdata.go b/src/pkg/mime/multipart/formdata.go index 287938557..5f3286565 100644 --- a/src/pkg/mime/multipart/formdata.go +++ b/src/pkg/mime/multipart/formdata.go @@ -30,21 +30,18 @@ func (r *multiReader) ReadForm(maxMemory int64) (f *Form, err os.Error) { maxValueBytes := int64(10 << 20) // 10 MB is a lot of text. for { p, err := r.NextPart() + if err == os.EOF { + break + } if err != nil { return nil, err } - if p == nil { - break - } name := p.FormName() if name == "" { continue } - var filename string - if p.dispositionParams != nil { - filename = p.dispositionParams["filename"] - } + filename := p.FileName() var b bytes.Buffer diff --git a/src/pkg/mime/multipart/formdata_test.go b/src/pkg/mime/multipart/formdata_test.go index b56e2a430..9424c3778 100644 --- a/src/pkg/mime/multipart/formdata_test.go +++ b/src/pkg/mime/multipart/formdata_test.go @@ -33,7 +33,7 @@ func TestReadForm(t *testing.T) { } fd = testFile(t, f.File["fileb"][0], "fileb.txt", filebContents) if _, ok := fd.(*os.File); !ok { - t.Error("file has unexpected underlying type %T", fd) + t.Errorf("file has unexpected underlying type %T", fd) } } diff --git a/src/pkg/mime/multipart/multipart.go b/src/pkg/mime/multipart/multipart.go index 60329fe17..9affa1126 100644 --- a/src/pkg/mime/multipart/multipart.go +++ b/src/pkg/mime/multipart/multipart.go @@ -26,14 +26,14 @@ import ( var headerRegexp *regexp.Regexp = regexp.MustCompile("^([a-zA-Z0-9\\-]+): *([^\r\n]+)") +var emptyParams = make(map[string]string) + // Reader is an iterator over parts in a MIME multipart body. // Reader's underlying parser consumes its input as needed. Seeking // isn't supported. type Reader interface { - // NextPart returns the next part in the multipart, or (nil, - // nil) on EOF. An error is returned if the underlying reader - // reports errors, or on truncated or otherwise malformed - // input. + // NextPart returns the next part in the multipart or an error. + // When there are no more parts, the error os.EOF is returned. NextPart() (*Part, os.Error) // ReadForm parses an entire multipart message whose parts have @@ -53,6 +53,7 @@ type Part struct { buffer *bytes.Buffer mr *multiReader + disposition string dispositionParams map[string]string } @@ -61,21 +62,33 @@ type Part struct { func (p *Part) FormName() string { // See http://tools.ietf.org/html/rfc2183 section 2 for EBNF // of Content-Disposition value format. - if p.dispositionParams != nil { - return p.dispositionParams["name"] - } - v := p.Header.Get("Content-Disposition") - if v == "" { - return "" + if p.dispositionParams == nil { + p.parseContentDisposition() } - if d, params := mime.ParseMediaType(v); d != "form-data" { + if p.disposition != "form-data" { return "" - } else { - p.dispositionParams = params } return p.dispositionParams["name"] } + +// FileName returns the filename parameter of the Part's +// Content-Disposition header. +func (p *Part) FileName() string { + if p.dispositionParams == nil { + p.parseContentDisposition() + } + return p.dispositionParams["filename"] +} + +func (p *Part) parseContentDisposition() { + v := p.Header.Get("Content-Disposition") + p.disposition, p.dispositionParams = mime.ParseMediaType(v) + if p.dispositionParams == nil { + p.dispositionParams = emptyParams + } +} + // NewReader creates a new multipart Reader reading from r using the // given MIME boundary. func NewReader(reader io.Reader, boundary string) Reader { @@ -207,9 +220,8 @@ func (mr *multiReader) NextPart() (*Part, os.Error) { } if hasPrefixThenNewline(line, mr.dashBoundaryDash) { - // Expected EOF (no error) - // TODO(bradfitz): should return an os.EOF error here, not using nil for errors - return nil, nil + // Expected EOF + return nil, os.EOF } if expectNewPart { diff --git a/src/pkg/mime/multipart/multipart_test.go b/src/pkg/mime/multipart/multipart_test.go index 16249146c..4ec3d30bd 100644 --- a/src/pkg/mime/multipart/multipart_test.go +++ b/src/pkg/mime/multipart/multipart_test.go @@ -56,24 +56,25 @@ func expectEq(t *testing.T, expected, actual, what string) { what, escapeString(actual), len(actual), escapeString(expected), len(expected)) } -func TestFormName(t *testing.T) { - p := new(Part) - p.Header = make(map[string][]string) - tests := [...][2]string{ - {`form-data; name="foo"`, "foo"}, - {` form-data ; name=foo`, "foo"}, - {`FORM-DATA;name="foo"`, "foo"}, - {` FORM-DATA ; name="foo"`, "foo"}, - {` FORM-DATA ; name="foo"`, "foo"}, - {` FORM-DATA ; name=foo`, "foo"}, - {` FORM-DATA ; filename="foo.txt"; name=foo; baz=quux`, "foo"}, - } - for _, test := range tests { +func TestNameAccessors(t *testing.T) { + tests := [...][3]string{ + {`form-data; name="foo"`, "foo", ""}, + {` form-data ; name=foo`, "foo", ""}, + {`FORM-DATA;name="foo"`, "foo", ""}, + {` FORM-DATA ; name="foo"`, "foo", ""}, + {` FORM-DATA ; name="foo"`, "foo", ""}, + {` FORM-DATA ; name=foo`, "foo", ""}, + {` FORM-DATA ; filename="foo.txt"; name=foo; baz=quux`, "foo", "foo.txt"}, + {` not-form-data ; filename="bar.txt"; name=foo; baz=quux`, "", "bar.txt"}, + } + for i, test := range tests { + p := &Part{Header: make(map[string][]string)} p.Header.Set("Content-Disposition", test[0]) - expected := test[1] - actual := p.FormName() - if actual != expected { - t.Errorf("expected \"%s\"; got: \"%s\"", expected, actual) + if g, e := p.FormName(), test[1]; g != e { + t.Errorf("test %d: FormName() = %q; want %q", i, g, e) + } + if g, e := p.FileName(), test[2]; g != e { + t.Errorf("test %d: FileName() = %q; want %q", i, g, e) } } } @@ -201,8 +202,8 @@ func testMultipart(t *testing.T, r io.Reader) { if part != nil { t.Error("Didn't expect a fifth part.") } - if err != nil { - t.Errorf("Unexpected error getting fifth part: %v", err) + if err != os.EOF { + t.Errorf("On fifth part expected os.EOF; got %v", err) } } @@ -246,8 +247,8 @@ func TestVariousTextLineEndings(t *testing.T) { if part != nil { t.Errorf("Unexpected part in test %d", testNum) } - if err != nil { - t.Errorf("Unexpected error in test %d: %v", testNum, err) + if err != os.EOF { + t.Errorf("On test %d expected os.EOF; got %v", testNum, err) } } @@ -306,6 +307,29 @@ Oh no, premature EOF! } } +func TestZeroLengthBody(t *testing.T) { + testBody := strings.Replace(` +This is a multi-part message. This line is ignored. +--MyBoundary +foo: bar + + +--MyBoundary-- +`, "\n", "\r\n", -1) + r := NewReader(strings.NewReader(testBody), "MyBoundary") + part, err := r.NextPart() + if err != nil { + t.Fatalf("didn't get a part") + } + n, err := io.Copy(ioutil.Discard, part) + if err != nil { + t.Errorf("error reading part: %v", err) + } + if n != 0 { + t.Errorf("read %d bytes; expected 0", n) + } +} + type slowReader struct { r io.Reader } diff --git a/src/pkg/mime/multipart/writer.go b/src/pkg/mime/multipart/writer.go new file mode 100644 index 000000000..b436dd012 --- /dev/null +++ b/src/pkg/mime/multipart/writer.go @@ -0,0 +1,153 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package multipart + +import ( + "bytes" + "crypto/rand" + "fmt" + "io" + "net/textproto" + "os" + "strings" +) + +// A Writer generates multipart messages. +type Writer struct { + w io.Writer + boundary string + lastpart *part +} + +// NewWriter returns a new multipart Writer with a random boundary, +// writing to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{ + w: w, + boundary: randomBoundary(), + } +} + +// Boundary returns the Writer's randomly selected boundary string. +func (w *Writer) Boundary() string { + return w.boundary +} + +// FormDataContentType returns the Content-Type for an HTTP +// multipart/form-data with this Writer's Boundary. +func (w *Writer) FormDataContentType() string { + return "multipart/form-data; boundary=" + w.boundary +} + +func randomBoundary() string { + var buf [30]byte + _, err := io.ReadFull(rand.Reader, buf[:]) + if err != nil { + panic(err) + } + return fmt.Sprintf("%x", buf[:]) +} + +// CreatePart creates a new multipart section with the provided +// header. The body of the part should be written to the returned +// Writer. After calling CreatePart, any previous part may no longer +// be written to. +func (w *Writer) CreatePart(header textproto.MIMEHeader) (io.Writer, os.Error) { + if w.lastpart != nil { + if err := w.lastpart.close(); err != nil { + return nil, err + } + } + var b bytes.Buffer + fmt.Fprintf(&b, "\r\n--%s\r\n", w.boundary) + // TODO(bradfitz): move this to textproto.MimeHeader.Write(w), have it sort + // and clean, like http.Header.Write(w) does. + for k, vv := range header { + for _, v := range vv { + fmt.Fprintf(&b, "%s: %s\r\n", k, v) + } + } + fmt.Fprintf(&b, "\r\n") + _, err := io.Copy(w.w, &b) + if err != nil { + return nil, err + } + p := &part{ + mw: w, + } + w.lastpart = p + return p, nil +} + +func escapeQuotes(s string) string { + s = strings.Replace(s, "\\", "\\\\", -1) + s = strings.Replace(s, "\"", "\\\"", -1) + return s +} + +// CreateFormFile is a convenience wrapper around CreatePart. It creates +// a new form-data header with the provided field name and file name. +func (w *Writer) CreateFormFile(fieldname, filename string) (io.Writer, os.Error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", + fmt.Sprintf(`form-data; name="%s"; filename="%s"`, + escapeQuotes(fieldname), escapeQuotes(filename))) + h.Set("Content-Type", "application/octet-stream") + return w.CreatePart(h) +} + +// CreateFormField calls CreatePart with a header using the +// given field name. +func (w *Writer) CreateFormField(fieldname string) (io.Writer, os.Error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", + fmt.Sprintf(`form-data; name="%s"`, escapeQuotes(fieldname))) + return w.CreatePart(h) +} + +// WriteField calls CreateFormField and then writes the given value. +func (w *Writer) WriteField(fieldname, value string) os.Error { + p, err := w.CreateFormField(fieldname) + if err != nil { + return err + } + _, err = p.Write([]byte(value)) + return err +} + +// Close finishes the multipart message and writes the trailing +// boundary end line to the output. +func (w *Writer) Close() os.Error { + if w.lastpart != nil { + if err := w.lastpart.close(); err != nil { + return err + } + w.lastpart = nil + } + _, err := fmt.Fprintf(w.w, "\r\n--%s--\r\n", w.boundary) + return err +} + +type part struct { + mw *Writer + closed bool + we os.Error // last error that occurred writing +} + +func (p *part) close() os.Error { + p.closed = true + return p.we +} + +func (p *part) Write(d []byte) (n int, err os.Error) { + if p.closed { + return 0, os.NewError("multipart: can't write to finished part") + } + n, err = p.mw.w.Write(d) + if err != nil { + p.we = err + } + return +} diff --git a/src/pkg/mime/multipart/writer_test.go b/src/pkg/mime/multipart/writer_test.go new file mode 100644 index 000000000..e6a04c388 --- /dev/null +++ b/src/pkg/mime/multipart/writer_test.go @@ -0,0 +1,71 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package multipart + +import ( + "bytes" + "io/ioutil" + "testing" +) + +func TestWriter(t *testing.T) { + fileContents := []byte("my file contents") + + var b bytes.Buffer + w := NewWriter(&b) + { + part, err := w.CreateFormFile("myfile", "my-file.txt") + if err != nil { + t.Fatalf("CreateFormFile: %v", err) + } + part.Write(fileContents) + err = w.WriteField("key", "val") + if err != nil { + t.Fatalf("WriteField: %v", err) + } + part.Write([]byte("val")) + err = w.Close() + if err != nil { + t.Fatalf("Close: %v", err) + } + } + + r := NewReader(&b, w.Boundary()) + + part, err := r.NextPart() + if err != nil { + t.Fatalf("part 1: %v", err) + } + if g, e := part.FormName(), "myfile"; g != e { + t.Errorf("part 1: want form name %q, got %q", e, g) + } + slurp, err := ioutil.ReadAll(part) + if err != nil { + t.Fatalf("part 1: ReadAll: %v", err) + } + if e, g := string(fileContents), string(slurp); e != g { + t.Errorf("part 1: want contents %q, got %q", e, g) + } + + part, err = r.NextPart() + if err != nil { + t.Fatalf("part 2: %v", err) + } + if g, e := part.FormName(), "key"; g != e { + t.Errorf("part 2: want form name %q, got %q", e, g) + } + slurp, err = ioutil.ReadAll(part) + if err != nil { + t.Fatalf("part 2: ReadAll: %v", err) + } + if e, g := "val", string(slurp); e != g { + t.Errorf("part 2: want contents %q, got %q", e, g) + } + + part, err = r.NextPart() + if part != nil || err == nil { + t.Fatalf("expected end of parts; got %v, %v", part, err) + } +} diff --git a/src/pkg/net/Makefile b/src/pkg/net/Makefile index 221871cb1..5472df392 100644 --- a/src/pkg/net/Makefile +++ b/src/pkg/net/Makefile @@ -10,6 +10,7 @@ GOFILES=\ dnsmsg.go\ fd_$(GOOS).go\ hosts.go\ + interface.go\ ip.go\ ipsock.go\ iprawsock.go\ @@ -23,36 +24,49 @@ GOFILES=\ unixsock.go\ GOFILES_freebsd=\ - newpollserver.go\ + dnsclient.go\ + dnsconfig.go\ fd.go\ file.go\ - dnsconfig.go\ - dnsclient.go\ + interface_bsd.go\ + newpollserver.go\ port.go\ + sendfile_stub.go\ + sock_bsd.go\ CGOFILES_freebsd=\ cgo_bsd.go\ cgo_unix.go\ GOFILES_darwin=\ - newpollserver.go\ + dnsclient.go\ + dnsconfig.go\ fd.go\ file.go\ - dnsconfig.go\ - dnsclient.go\ + interface_bsd.go\ + newpollserver.go\ port.go\ + sendfile_stub.go\ + sock_bsd.go\ CGOFILES_darwin=\ cgo_bsd.go\ cgo_unix.go\ - + GOFILES_linux=\ - newpollserver.go\ + dnsclient.go\ + dnsconfig.go\ fd.go\ file.go\ - dnsconfig.go\ - dnsclient.go\ + interface_linux.go\ + newpollserver.go\ port.go\ + sendfile_linux.go\ + sock_linux.go\ + +GOFILES_plan9=\ + interface_stub.go\ + sendfile_stub.go\ ifeq ($(GOARCH),arm) # ARM has no cgo, so use the stubs. @@ -65,8 +79,11 @@ endif GOFILES_windows=\ cgo_stub.go\ - resolv_windows.go\ file_windows.go\ + interface_stub.go\ + resolv_windows.go\ + sendfile_stub.go\ + sock_windows.go\ GOFILES+=$(GOFILES_$(GOOS)) ifneq ($(CGOFILES_$(GOOS)),) diff --git a/src/pkg/net/dial.go b/src/pkg/net/dial.go index 16896b426..ead775fe6 100644 --- a/src/pkg/net/dial.go +++ b/src/pkg/net/dial.go @@ -60,7 +60,7 @@ func Dial(net, addr string) (c Conn, err os.Error) { return c, nil case "ip", "ip4", "ip6": var ra *IPAddr - if ra, err = ResolveIPAddr(raddr); err != nil { + if ra, err = ResolveIPAddr(net, raddr); err != nil { goto Error } c, err := DialIP(net, nil, ra) @@ -139,12 +139,13 @@ func ListenPacket(net, laddr string) (c PacketConn, err os.Error) { return c, nil } - if i := last(net, ':'); i > 0 { - switch net[0:i] { + var rawnet string + if rawnet, _, err = splitNetProto(net); err != nil { + switch rawnet { case "ip", "ip4", "ip6": var la *IPAddr if laddr != "" { - if la, err = ResolveIPAddr(laddr); err != nil { + if la, err = ResolveIPAddr(rawnet, laddr); err != nil { return nil, err } } diff --git a/src/pkg/net/dialgoogle_test.go b/src/pkg/net/dialgoogle_test.go index c25089ba4..9ad1770da 100644 --- a/src/pkg/net/dialgoogle_test.go +++ b/src/pkg/net/dialgoogle_test.go @@ -41,20 +41,6 @@ func doDial(t *testing.T, network, addr string) { fd.Close() } -var googleaddrs = []string{ - "%d.%d.%d.%d:80", - "www.google.com:80", - "%d.%d.%d.%d:http", - "www.google.com:http", - "%03d.%03d.%03d.%03d:0080", - "[::ffff:%d.%d.%d.%d]:80", - "[::ffff:%02x%02x:%02x%02x]:80", - "[0:0:0:0:0000:ffff:%d.%d.%d.%d]:80", - "[0:0:0:0:000000:ffff:%d.%d.%d.%d]:80", - "[0:0:0:0:0:ffff::%d.%d.%d.%d]:80", - "[2001:4860:0:2001::68]:80", // ipv6.google.com; removed if ipv6 flag not set -} - func TestLookupCNAME(t *testing.T) { if testing.Short() { // Don't use external network. @@ -67,16 +53,25 @@ func TestLookupCNAME(t *testing.T) { } } -func TestDialGoogle(t *testing.T) { +var googleaddrsipv4 = []string{ + "%d.%d.%d.%d:80", + "www.google.com:80", + "%d.%d.%d.%d:http", + "www.google.com:http", + "%03d.%03d.%03d.%03d:0080", + "[::ffff:%d.%d.%d.%d]:80", + "[::ffff:%02x%02x:%02x%02x]:80", + "[0:0:0:0:0000:ffff:%d.%d.%d.%d]:80", + "[0:0:0:0:000000:ffff:%d.%d.%d.%d]:80", + "[0:0:0:0:0:ffff::%d.%d.%d.%d]:80", +} + +func TestDialGoogleIPv4(t *testing.T) { if testing.Short() { // Don't use external network. t.Logf("skipping external network test during -short") return } - // If no ipv6 tunnel, don't try the last address. - if !*ipv6 { - googleaddrs[len(googleaddrs)-1] = "" - } // Insert an actual IPv4 address for google.com // into the table. @@ -95,14 +90,14 @@ func TestDialGoogle(t *testing.T) { t.Fatalf("no IPv4 addresses for www.google.com") } - for i, s := range googleaddrs { + for i, s := range googleaddrsipv4 { if strings.Contains(s, "%") { - googleaddrs[i] = fmt.Sprintf(s, ip[0], ip[1], ip[2], ip[3]) + googleaddrsipv4[i] = fmt.Sprintf(s, ip[0], ip[1], ip[2], ip[3]) } } - for i := 0; i < len(googleaddrs); i++ { - addr := googleaddrs[i] + for i := 0; i < len(googleaddrsipv4); i++ { + addr := googleaddrsipv4[i] if addr == "" { continue } @@ -110,20 +105,65 @@ func TestDialGoogle(t *testing.T) { doDial(t, "tcp", addr) if addr[0] != '[' { doDial(t, "tcp4", addr) - - if !preferIPv4 { - // make sure preferIPv4 flag works. - preferIPv4 = true + if supportsIPv6 { + // make sure syscall.SocketDisableIPv6 flag works. syscall.SocketDisableIPv6 = true + doDial(t, "tcp", addr) doDial(t, "tcp4", addr) syscall.SocketDisableIPv6 = false - preferIPv4 = false } } + } +} + +var googleaddrsipv6 = []string{ + "[%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x]:80", + "ipv6.google.com:80", + "[%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x]:http", + "ipv6.google.com:http", +} + +func TestDialGoogleIPv6(t *testing.T) { + if testing.Short() { + // Don't use external network. + t.Logf("skipping external network test during -short") + return + } + // Only run tcp6 if the kernel will take it. + if !*ipv6 || !supportsIPv6 { + return + } + + // Insert an actual IPv6 address for ipv6.google.com + // into the table. + addrs, err := LookupIP("ipv6.google.com") + if err != nil { + t.Fatalf("lookup ipv6.google.com: %v", err) + } + var ip IP + for _, addr := range addrs { + if x := addr.To16(); x != nil { + ip = x + break + } + } + if ip == nil { + t.Fatalf("no IPv6 addresses for ipv6.google.com") + } + + for i, s := range googleaddrsipv6 { + if strings.Contains(s, "%") { + googleaddrsipv6[i] = fmt.Sprintf(s, ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], ip[8], ip[9], ip[10], ip[11], ip[12], ip[13], ip[14], ip[15]) + } + } - // Only run tcp6 if the kernel will take it. - if kernelSupportsIPv6() { - doDial(t, "tcp6", addr) + for i := 0; i < len(googleaddrsipv6); i++ { + addr := googleaddrsipv6[i] + if addr == "" { + continue } + t.Logf("-- %s --", addr) + doDial(t, "tcp", addr) + doDial(t, "tcp6", addr) } } diff --git a/src/pkg/net/dnsclient.go b/src/pkg/net/dnsclient.go index 89f2409bf..ae9ca8430 100644 --- a/src/pkg/net/dnsclient.go +++ b/src/pkg/net/dnsclient.go @@ -121,15 +121,19 @@ func answer(name, server string, dns *dnsMsg, qtype uint16) (cname string, addrs Cname: for cnameloop := 0; cnameloop < 10; cnameloop++ { addrs = addrs[0:0] - for i := 0; i < len(dns.answer); i++ { - rr := dns.answer[i] + for _, rr := range dns.answer { + if _, justHeader := rr.(*dnsRR_Header); justHeader { + // Corrupt record: we only have a + // header. That header might say it's + // of type qtype, but we don't + // actually have it. Skip. + continue + } h := rr.Header() if h.Class == dnsClassINET && h.Name == name { switch h.Rrtype { case qtype: - n := len(addrs) - addrs = addrs[0 : n+1] - addrs[n] = rr + addrs = append(addrs, rr) case dnsTypeCNAME: // redirect to cname name = rr.(*dnsRR_CNAME).Cname @@ -181,8 +185,7 @@ func tryOneName(cfg *dnsConfig, name string, qtype uint16) (cname string, addrs func convertRR_A(records []dnsRR) []IP { addrs := make([]IP, len(records)) - for i := 0; i < len(records); i++ { - rr := records[i] + for i, rr := range records { a := rr.(*dnsRR_A).A addrs[i] = IPv4(byte(a>>24), byte(a>>16), byte(a>>8), byte(a)) } @@ -191,8 +194,7 @@ func convertRR_A(records []dnsRR) []IP { func convertRR_AAAA(records []dnsRR) []IP { addrs := make([]IP, len(records)) - for i := 0; i < len(records); i++ { - rr := records[i] + for i, rr := range records { a := make(IP, 16) copy(a, rr.(*dnsRR_AAAA).AAAA[:]) addrs[i] = a @@ -384,9 +386,7 @@ func goLookupCNAME(name string) (cname string, err os.Error) { if err != nil { return } - if len(rr) >= 0 { - cname = rr[0].(*dnsRR_CNAME).Cname - } + cname = rr[0].(*dnsRR_CNAME).Cname return } @@ -398,10 +398,49 @@ type SRV struct { Weight uint16 } +// byPriorityWeight sorts SRV records by ascending priority and weight. +type byPriorityWeight []*SRV + +func (s byPriorityWeight) Len() int { return len(s) } + +func (s byPriorityWeight) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func (s byPriorityWeight) Less(i, j int) bool { + return s[i].Priority < s[j].Priority || + (s[i].Priority == s[j].Priority && s[i].Weight < s[j].Weight) +} + +// shuffleSRVByWeight shuffles SRV records by weight using the algorithm +// described in RFC 2782. +func shuffleSRVByWeight(addrs []*SRV) { + sum := 0 + for _, addr := range addrs { + sum += int(addr.Weight) + } + for sum > 0 && len(addrs) > 1 { + s := 0 + n := rand.Intn(sum + 1) + for i := range addrs { + s += int(addrs[i].Weight) + if s >= n { + if i > 0 { + t := addrs[i] + copy(addrs[1:i+1], addrs[0:i]) + addrs[0] = t + } + break + } + } + sum -= int(addrs[0].Weight) + addrs = addrs[1:] + } +} + // LookupSRV tries to resolve an SRV query of the given service, // protocol, and domain name, as specified in RFC 2782. In most cases // the proto argument can be the same as the corresponding -// Addr.Network(). +// Addr.Network(). The returned records are sorted by priority +// and randomized by weight within a priority. func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err os.Error) { target := "_" + service + "._" + proto + "." + name var records []dnsRR @@ -410,10 +449,19 @@ func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err os. return } addrs = make([]*SRV, len(records)) - for i := 0; i < len(records); i++ { - r := records[i].(*dnsRR_SRV) + for i, rr := range records { + r := rr.(*dnsRR_SRV) addrs[i] = &SRV{r.Target, r.Port, r.Priority, r.Weight} } + sort.Sort(byPriorityWeight(addrs)) + i := 0 + for j := 1; j < len(addrs); j++ { + if addrs[i].Priority != addrs[j].Priority { + shuffleSRVByWeight(addrs[i:j]) + i = j + } + } + shuffleSRVByWeight(addrs[i:len(addrs)]) return } diff --git a/src/pkg/net/dnsconfig.go b/src/pkg/net/dnsconfig.go index 26f0e04e9..54e334342 100644 --- a/src/pkg/net/dnsconfig.go +++ b/src/pkg/net/dnsconfig.go @@ -30,7 +30,6 @@ func (e *DNSConfigError) String() string { func (e *DNSConfigError) Timeout() bool { return false } func (e *DNSConfigError) Temporary() bool { return false } - // See resolv.conf(5) on a Linux machine. // TODO(rsc): Supposed to call uname() and chop the beginning // of the host name to get the default search domain. diff --git a/src/pkg/net/dnsmsg.go b/src/pkg/net/dnsmsg.go index 7b8e5c6d3..0ba69a0ce 100644 --- a/src/pkg/net/dnsmsg.go +++ b/src/pkg/net/dnsmsg.go @@ -117,7 +117,6 @@ type dnsRR interface { Header() *dnsRR_Header } - // Specific DNS RR formats for each query type. type dnsRR_CNAME struct { @@ -715,24 +714,35 @@ func (dns *dnsMsg) Unpack(msg []byte) bool { // Arrays. dns.question = make([]dnsQuestion, dh.Qdcount) - dns.answer = make([]dnsRR, dh.Ancount) - dns.ns = make([]dnsRR, dh.Nscount) - dns.extra = make([]dnsRR, dh.Arcount) + dns.answer = make([]dnsRR, 0, dh.Ancount) + dns.ns = make([]dnsRR, 0, dh.Nscount) + dns.extra = make([]dnsRR, 0, dh.Arcount) + + var rec dnsRR for i := 0; i < len(dns.question); i++ { off, ok = unpackStruct(&dns.question[i], msg, off) } - for i := 0; i < len(dns.answer); i++ { - dns.answer[i], off, ok = unpackRR(msg, off) - } - for i := 0; i < len(dns.ns); i++ { - dns.ns[i], off, ok = unpackRR(msg, off) + for i := 0; i < int(dh.Ancount); i++ { + rec, off, ok = unpackRR(msg, off) + if !ok { + return false + } + dns.answer = append(dns.answer, rec) } - for i := 0; i < len(dns.extra); i++ { - dns.extra[i], off, ok = unpackRR(msg, off) + for i := 0; i < int(dh.Nscount); i++ { + rec, off, ok = unpackRR(msg, off) + if !ok { + return false + } + dns.ns = append(dns.ns, rec) } - if !ok { - return false + for i := 0; i < int(dh.Arcount); i++ { + rec, off, ok = unpackRR(msg, off) + if !ok { + return false + } + dns.extra = append(dns.extra, rec) } // if off != len(msg) { // println("extra bytes in dns packet", off, "<", len(msg)); diff --git a/src/pkg/net/dnsmsg_test.go b/src/pkg/net/dnsmsg_test.go new file mode 100644 index 000000000..20c9f02b0 --- /dev/null +++ b/src/pkg/net/dnsmsg_test.go @@ -0,0 +1,107 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "encoding/hex" + "runtime" + "testing" +) + +func TestDNSParseSRVReply(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + data, err := hex.DecodeString(dnsSRVReply) + if err != nil { + t.Fatal(err) + } + msg := new(dnsMsg) + ok := msg.Unpack(data) + if !ok { + t.Fatalf("unpacking packet failed") + } + if g, e := len(msg.answer), 5; g != e { + t.Errorf("len(msg.answer) = %d; want %d", g, e) + } + for idx, rr := range msg.answer { + if g, e := rr.Header().Rrtype, uint16(dnsTypeSRV); g != e { + t.Errorf("rr[%d].Header().Rrtype = %d; want %d", idx, g, e) + } + if _, ok := rr.(*dnsRR_SRV); !ok { + t.Errorf("answer[%d] = %T; want *dnsRR_SRV", idx, rr) + } + } + _, addrs, err := answer("_xmpp-server._tcp.google.com.", "foo:53", msg, uint16(dnsTypeSRV)) + if err != nil { + t.Fatalf("answer: %v", err) + } + if g, e := len(addrs), 5; g != e { + t.Errorf("len(addrs) = %d; want %d", g, e) + t.Logf("addrs = %#v", addrs) + } +} + +func TestDNSParseCorruptSRVReply(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + data, err := hex.DecodeString(dnsSRVCorruptReply) + if err != nil { + t.Fatal(err) + } + msg := new(dnsMsg) + ok := msg.Unpack(data) + if !ok { + t.Fatalf("unpacking packet failed") + } + if g, e := len(msg.answer), 5; g != e { + t.Errorf("len(msg.answer) = %d; want %d", g, e) + } + for idx, rr := range msg.answer { + if g, e := rr.Header().Rrtype, uint16(dnsTypeSRV); g != e { + t.Errorf("rr[%d].Header().Rrtype = %d; want %d", idx, g, e) + } + if idx == 4 { + if _, ok := rr.(*dnsRR_Header); !ok { + t.Errorf("answer[%d] = %T; want *dnsRR_Header", idx, rr) + } + } else { + if _, ok := rr.(*dnsRR_SRV); !ok { + t.Errorf("answer[%d] = %T; want *dnsRR_SRV", idx, rr) + } + } + } + _, addrs, err := answer("_xmpp-server._tcp.google.com.", "foo:53", msg, uint16(dnsTypeSRV)) + if err != nil { + t.Fatalf("answer: %v", err) + } + if g, e := len(addrs), 4; g != e { + t.Errorf("len(addrs) = %d; want %d", g, e) + t.Logf("addrs = %#v", addrs) + } +} + +// Valid DNS SRV reply +const dnsSRVReply = "0901818000010005000000000c5f786d70702d736572766572045f74637006676f6f67" + + "6c6503636f6d0000210001c00c002100010000012c00210014000014950c786d70702d" + + "73657276657234016c06676f6f676c6503636f6d00c00c002100010000012c00210014" + + "000014950c786d70702d73657276657232016c06676f6f676c6503636f6d00c00c0021" + + "00010000012c00210014000014950c786d70702d73657276657233016c06676f6f676c" + + "6503636f6d00c00c002100010000012c00200005000014950b786d70702d7365727665" + + "72016c06676f6f676c6503636f6d00c00c002100010000012c00210014000014950c78" + + "6d70702d73657276657231016c06676f6f676c6503636f6d00" + +// Corrupt DNS SRV reply, with its final RR having a bogus length +// (perhaps it was truncated, or it's malicious) The mutation is the +// capital "FF" below, instead of the proper "21". +const dnsSRVCorruptReply = "0901818000010005000000000c5f786d70702d736572766572045f74637006676f6f67" + + "6c6503636f6d0000210001c00c002100010000012c00210014000014950c786d70702d" + + "73657276657234016c06676f6f676c6503636f6d00c00c002100010000012c00210014" + + "000014950c786d70702d73657276657232016c06676f6f676c6503636f6d00c00c0021" + + "00010000012c00210014000014950c786d70702d73657276657233016c06676f6f676c" + + "6503636f6d00c00c002100010000012c00200005000014950b786d70702d7365727665" + + "72016c06676f6f676c6503636f6d00c00c002100010000012c00FF0014000014950c78" + + "6d70702d73657276657231016c06676f6f676c6503636f6d00" diff --git a/src/pkg/net/fd_linux.go b/src/pkg/net/fd_linux.go index dcf65c014..70fc344b2 100644 --- a/src/pkg/net/fd_linux.go +++ b/src/pkg/net/fd_linux.go @@ -117,6 +117,17 @@ func (p *pollster) DelFD(fd int, mode int) { } else { p.StopWaiting(fd, writeFlags) } + + // Discard any queued up events. + i := 0 + for i < len(p.waitEvents) { + if fd == int(p.waitEvents[i].Fd) { + copy(p.waitEvents[i:], p.waitEvents[i+1:]) + p.waitEvents = p.waitEvents[:len(p.waitEvents)-1] + } else { + i++ + } + } } func (p *pollster) WaitFD(s *pollServer, nsec int64) (fd int, mode int, err os.Error) { diff --git a/src/pkg/net/fd_windows.go b/src/pkg/net/fd_windows.go index c2f736cc1..9ed7801d2 100644 --- a/src/pkg/net/fd_windows.go +++ b/src/pkg/net/fd_windows.go @@ -84,7 +84,7 @@ func (o *bufOp) Init(fd *netFD, buf []byte) { } } -// resultSrv will retreive all io completion results from +// resultSrv will retrieve all io completion results from // iocp and send them to the correspondent waiting client // goroutine via channel supplied in the request. type resultSrv struct { @@ -513,7 +513,7 @@ func (fd *netFD) accept(toAddr func(syscall.Sockaddr) Addr) (nfd *netFD, err os. return nfd, nil } -// Not implemeted functions. +// Unimplemented functions. func (fd *netFD) dup() (f *os.File, err os.Error) { // TODO: Implement this diff --git a/src/pkg/net/file_test.go b/src/pkg/net/file_test.go index 1ec05fdee..bd1e2c9d7 100644 --- a/src/pkg/net/file_test.go +++ b/src/pkg/net/file_test.go @@ -62,7 +62,7 @@ func TestFileListener(t *testing.T) { } testFileListener(t, "tcp", "127.0.0.1") testFileListener(t, "tcp", "127.0.0.1") - if kernelSupportsIPv6() { + if supportsIPv6 && supportsIPv4map { testFileListener(t, "tcp", "[::ffff:127.0.0.1]") testFileListener(t, "tcp", "127.0.0.1") testFileListener(t, "tcp", "[::ffff:127.0.0.1]") @@ -121,8 +121,10 @@ func TestFilePacketConn(t *testing.T) { } testFilePacketConnListen(t, "udp", "127.0.0.1:0") testFilePacketConnDial(t, "udp", "127.0.0.1:12345") - if kernelSupportsIPv6() { + if supportsIPv6 { testFilePacketConnListen(t, "udp", "[::1]:0") + } + if supportsIPv6 && supportsIPv4map { testFilePacketConnDial(t, "udp", "[::ffff:127.0.0.1]:12345") } if syscall.OS == "linux" { diff --git a/src/pkg/net/interface.go b/src/pkg/net/interface.go new file mode 100644 index 000000000..f622487ab --- /dev/null +++ b/src/pkg/net/interface.go @@ -0,0 +1,89 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Network interface identification + +package net + +import ( + "bytes" + "fmt" + "os" +) + +// A HardwareAddr represents a physical hardware address. +type HardwareAddr []byte + +func (a HardwareAddr) String() string { + var buf bytes.Buffer + for i, b := range a { + if i > 0 { + buf.WriteByte(':') + } + fmt.Fprintf(&buf, "%02x", b) + } + return buf.String() +} + +// Interface represents a mapping between network interface name +// and index. It also represents network interface facility +// information. +type Interface struct { + Index int // positive integer that starts at one, zero is never used + MTU int // maximum transmission unit + Name string // e.g., "en0", "lo0", "eth0.100" + HardwareAddr HardwareAddr // IEEE MAC-48, EUI-48 and EUI-64 form + rawFlags int +} + +// Addrs returns interface addresses for a specific interface. +func (ifi *Interface) Addrs() ([]Addr, os.Error) { + if ifi == nil { + return nil, os.NewError("net: invalid interface") + } + return interfaceAddrTable(ifi.Index) +} + +// Interfaces returns a list of the systems's network interfaces. +func Interfaces() ([]Interface, os.Error) { + return interfaceTable(0) +} + +// InterfaceAddrs returns a list of the system's network interface +// addresses. +func InterfaceAddrs() ([]Addr, os.Error) { + return interfaceAddrTable(0) +} + +// InterfaceByIndex returns the interface specified by index. +func InterfaceByIndex(index int) (*Interface, os.Error) { + if index <= 0 { + return nil, os.NewError("net: invalid interface index") + } + ift, err := interfaceTable(index) + if err != nil { + return nil, err + } + for _, ifi := range ift { + return &ifi, nil + } + return nil, os.NewError("net: no such interface") +} + +// InterfaceByName returns the interface specified by name. +func InterfaceByName(name string) (*Interface, os.Error) { + if name == "" { + return nil, os.NewError("net: invalid interface name") + } + ift, err := interfaceTable(0) + if err != nil { + return nil, err + } + for _, ifi := range ift { + if name == ifi.Name { + return &ifi, nil + } + } + return nil, os.NewError("net: no such interface") +} diff --git a/src/pkg/net/interface_bsd.go b/src/pkg/net/interface_bsd.go new file mode 100644 index 000000000..141b95b38 --- /dev/null +++ b/src/pkg/net/interface_bsd.go @@ -0,0 +1,195 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Network interface identification for BSD variants + +package net + +import ( + "os" + "syscall" + "unsafe" +) + +// IsUp returns true if ifi is up. +func (ifi *Interface) IsUp() bool { + if ifi == nil { + return false + } + return ifi.rawFlags&syscall.IFF_UP != 0 +} + +// IsLoopback returns true if ifi is a loopback interface. +func (ifi *Interface) IsLoopback() bool { + if ifi == nil { + return false + } + return ifi.rawFlags&syscall.IFF_LOOPBACK != 0 +} + +// CanBroadcast returns true if ifi supports a broadcast access +// capability. +func (ifi *Interface) CanBroadcast() bool { + if ifi == nil { + return false + } + return ifi.rawFlags&syscall.IFF_BROADCAST != 0 +} + +// IsPointToPoint returns true if ifi belongs to a point-to-point +// link. +func (ifi *Interface) IsPointToPoint() bool { + if ifi == nil { + return false + } + return ifi.rawFlags&syscall.IFF_POINTOPOINT != 0 +} + +// CanMulticast returns true if ifi supports a multicast access +// capability. +func (ifi *Interface) CanMulticast() bool { + if ifi == nil { + return false + } + return ifi.rawFlags&syscall.IFF_MULTICAST != 0 +} + +// If the ifindex is zero, interfaceTable returns mappings of all +// network interfaces. Otheriwse it returns a mapping of a specific +// interface. +func interfaceTable(ifindex int) ([]Interface, os.Error) { + var ( + tab []byte + e int + msgs []syscall.RoutingMessage + ift []Interface + ) + + tab, e = syscall.RouteRIB(syscall.NET_RT_IFLIST, ifindex) + if e != 0 { + return nil, os.NewSyscallError("route rib", e) + } + + msgs, e = syscall.ParseRoutingMessage(tab) + if e != 0 { + return nil, os.NewSyscallError("route message", e) + } + + for _, m := range msgs { + switch v := m.(type) { + case *syscall.InterfaceMessage: + if ifindex == 0 || ifindex == int(v.Header.Index) { + ifi, err := newLink(v) + if err != nil { + return nil, err + } + ift = append(ift, ifi...) + } + } + } + + return ift, nil +} + +func newLink(m *syscall.InterfaceMessage) ([]Interface, os.Error) { + var ift []Interface + + sas, e := syscall.ParseRoutingSockaddr(m) + if e != 0 { + return nil, os.NewSyscallError("route sockaddr", e) + } + + for _, s := range sas { + switch v := s.(type) { + case *syscall.SockaddrDatalink: + // NOTE: SockaddrDatalink.Data is minimum work area, + // can be larger. + m.Data = m.Data[unsafe.Offsetof(v.Data):] + ifi := Interface{Index: int(m.Header.Index), rawFlags: int(m.Header.Flags)} + var name [syscall.IFNAMSIZ]byte + for i := 0; i < int(v.Nlen); i++ { + name[i] = byte(m.Data[i]) + } + ifi.Name = string(name[:v.Nlen]) + ifi.MTU = int(m.Header.Data.Mtu) + addr := make([]byte, v.Alen) + for i := 0; i < int(v.Alen); i++ { + addr[i] = byte(m.Data[int(v.Nlen)+i]) + } + ifi.HardwareAddr = addr[:v.Alen] + ift = append(ift, ifi) + } + } + + return ift, nil +} + +// If the ifindex is zero, interfaceAddrTable returns addresses +// for all network interfaces. Otherwise it returns addresses +// for a specific interface. +func interfaceAddrTable(ifindex int) ([]Addr, os.Error) { + var ( + tab []byte + e int + msgs []syscall.RoutingMessage + ifat []Addr + ) + + tab, e = syscall.RouteRIB(syscall.NET_RT_IFLIST, ifindex) + if e != 0 { + return nil, os.NewSyscallError("route rib", e) + } + + msgs, e = syscall.ParseRoutingMessage(tab) + if e != 0 { + return nil, os.NewSyscallError("route message", e) + } + + for _, m := range msgs { + switch v := m.(type) { + case *syscall.InterfaceAddrMessage: + if ifindex == 0 || ifindex == int(v.Header.Index) { + ifa, err := newAddr(v) + if err != nil { + return nil, err + } + ifat = append(ifat, ifa...) + } + } + } + + return ifat, nil +} + +func newAddr(m *syscall.InterfaceAddrMessage) ([]Addr, os.Error) { + var ifat []Addr + + sas, e := syscall.ParseRoutingSockaddr(m) + if e != 0 { + return nil, os.NewSyscallError("route sockaddr", e) + } + + for _, s := range sas { + var ifa IPAddr + switch v := s.(type) { + case *syscall.SockaddrInet4: + ifa.IP = IPv4(v.Addr[0], v.Addr[1], v.Addr[2], v.Addr[3]) + case *syscall.SockaddrInet6: + ifa.IP = make(IP, IPv6len) + copy(ifa.IP, v.Addr[:]) + // NOTE: KAME based IPv6 protcol stack usually embeds + // the interface index in the interface-local or link- + // local address as the kernel-internal form. + if ifa.IP.IsLinkLocalUnicast() || + ifa.IP.IsInterfaceLocalMulticast() || + ifa.IP.IsLinkLocalMulticast() { + // remove embedded scope zone ID + ifa.IP[2], ifa.IP[3] = 0, 0 + } + } + ifat = append(ifat, ifa.toAddr()) + } + + return ifat, nil +} diff --git a/src/pkg/net/interface_linux.go b/src/pkg/net/interface_linux.go new file mode 100644 index 000000000..5c9657834 --- /dev/null +++ b/src/pkg/net/interface_linux.go @@ -0,0 +1,208 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Network interface identification for Linux + +package net + +import ( + "os" + "syscall" + "unsafe" +) + +// IsUp returns true if ifi is up. +func (ifi *Interface) IsUp() bool { + if ifi == nil { + return false + } + return ifi.rawFlags&syscall.IFF_UP != 0 +} + +// IsLoopback returns true if ifi is a loopback interface. +func (ifi *Interface) IsLoopback() bool { + if ifi == nil { + return false + } + return ifi.rawFlags&syscall.IFF_LOOPBACK != 0 +} + +// CanBroadcast returns true if ifi supports a broadcast access +// capability. +func (ifi *Interface) CanBroadcast() bool { + if ifi == nil { + return false + } + return ifi.rawFlags&syscall.IFF_BROADCAST != 0 +} + +// IsPointToPoint returns true if ifi belongs to a point-to-point +// link. +func (ifi *Interface) IsPointToPoint() bool { + if ifi == nil { + return false + } + return ifi.rawFlags&syscall.IFF_POINTOPOINT != 0 +} + +// CanMulticast returns true if ifi supports a multicast access +// capability. +func (ifi *Interface) CanMulticast() bool { + if ifi == nil { + return false + } + return ifi.rawFlags&syscall.IFF_MULTICAST != 0 +} + +// If the ifindex is zero, interfaceTable returns mappings of all +// network interfaces. Otheriwse it returns a mapping of a specific +// interface. +func interfaceTable(ifindex int) ([]Interface, os.Error) { + var ( + ift []Interface + tab []byte + msgs []syscall.NetlinkMessage + e int + ) + + tab, e = syscall.NetlinkRIB(syscall.RTM_GETLINK, syscall.AF_UNSPEC) + if e != 0 { + return nil, os.NewSyscallError("netlink rib", e) + } + + msgs, e = syscall.ParseNetlinkMessage(tab) + if e != 0 { + return nil, os.NewSyscallError("netlink message", e) + } + + for _, m := range msgs { + switch m.Header.Type { + case syscall.NLMSG_DONE: + goto done + case syscall.RTM_NEWLINK: + ifim := (*syscall.IfInfomsg)(unsafe.Pointer(&m.Data[0])) + if ifindex == 0 || ifindex == int(ifim.Index) { + attrs, e := syscall.ParseNetlinkRouteAttr(&m) + if e != 0 { + return nil, os.NewSyscallError("netlink routeattr", e) + } + ifi := newLink(attrs, ifim) + ift = append(ift, ifi) + } + } + } + +done: + return ift, nil +} + +func newLink(attrs []syscall.NetlinkRouteAttr, ifim *syscall.IfInfomsg) Interface { + ifi := Interface{Index: int(ifim.Index), rawFlags: int(ifim.Flags)} + for _, a := range attrs { + switch a.Attr.Type { + case syscall.IFLA_ADDRESS: + var nonzero bool + for _, b := range a.Value { + if b != 0 { + nonzero = true + } + } + if nonzero { + ifi.HardwareAddr = a.Value[:] + } + case syscall.IFLA_IFNAME: + ifi.Name = string(a.Value[:]) + case syscall.IFLA_MTU: + ifi.MTU = int(uint32(a.Value[3])<<24 | uint32(a.Value[2])<<16 | uint32(a.Value[1])<<8 | uint32(a.Value[0])) + } + } + return ifi +} + +// If the ifindex is zero, interfaceAddrTable returns addresses +// for all network interfaces. Otherwise it returns addresses +// for a specific interface. +func interfaceAddrTable(ifindex int) ([]Addr, os.Error) { + var ( + ifat4 []Addr + ifat6 []Addr + tab []byte + msgs4 []syscall.NetlinkMessage + msgs6 []syscall.NetlinkMessage + e int + err os.Error + ) + + tab, e = syscall.NetlinkRIB(syscall.RTM_GETADDR, syscall.AF_INET) + if e != 0 { + return nil, os.NewSyscallError("netlink rib", e) + } + msgs4, e = syscall.ParseNetlinkMessage(tab) + if e != 0 { + return nil, os.NewSyscallError("netlink message", e) + } + ifat4, err = addrTable(msgs4, ifindex) + if err != nil { + return nil, err + } + + tab, e = syscall.NetlinkRIB(syscall.RTM_GETADDR, syscall.AF_INET6) + if e != 0 { + return nil, os.NewSyscallError("netlink rib", e) + } + msgs6, e = syscall.ParseNetlinkMessage(tab) + if e != 0 { + return nil, os.NewSyscallError("netlink message", e) + } + ifat6, err = addrTable(msgs6, ifindex) + if err != nil { + return nil, err + } + + return append(ifat4, ifat6...), nil +} + +func addrTable(msgs []syscall.NetlinkMessage, ifindex int) ([]Addr, os.Error) { + var ifat []Addr + + for _, m := range msgs { + switch m.Header.Type { + case syscall.NLMSG_DONE: + goto done + case syscall.RTM_NEWADDR: + ifam := (*syscall.IfAddrmsg)(unsafe.Pointer(&m.Data[0])) + if ifindex == 0 || ifindex == int(ifam.Index) { + attrs, e := syscall.ParseNetlinkRouteAttr(&m) + if e != 0 { + return nil, os.NewSyscallError("netlink routeattr", e) + } + ifat = append(ifat, newAddr(attrs, int(ifam.Family))...) + } + } + } + +done: + return ifat, nil +} + +func newAddr(attrs []syscall.NetlinkRouteAttr, family int) []Addr { + var ifat []Addr + + for _, a := range attrs { + switch a.Attr.Type { + case syscall.IFA_ADDRESS: + ifa := IPAddr{} + switch family { + case syscall.AF_INET: + ifa.IP = IPv4(a.Value[0], a.Value[1], a.Value[2], a.Value[3]) + case syscall.AF_INET6: + ifa.IP = make(IP, IPv6len) + copy(ifa.IP, a.Value[:]) + } + ifat = append(ifat, ifa.toAddr()) + } + } + + return ifat +} diff --git a/src/pkg/net/interface_stub.go b/src/pkg/net/interface_stub.go new file mode 100644 index 000000000..feb871bb5 --- /dev/null +++ b/src/pkg/net/interface_stub.go @@ -0,0 +1,51 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Network interface identification + +package net + +import "os" + +// IsUp returns true if ifi is up. +func (ifi *Interface) IsUp() bool { + return false +} + +// IsLoopback returns true if ifi is a loopback interface. +func (ifi *Interface) IsLoopback() bool { + return false +} + +// CanBroadcast returns true if ifi supports a broadcast access +// capability. +func (ifi *Interface) CanBroadcast() bool { + return false +} + +// IsPointToPoint returns true if ifi belongs to a point-to-point +// link. +func (ifi *Interface) IsPointToPoint() bool { + return false +} + +// CanMulticast returns true if ifi supports a multicast access +// capability. +func (ifi *Interface) CanMulticast() bool { + return false +} + +// If the ifindex is zero, interfaceTable returns mappings of all +// network interfaces. Otheriwse it returns a mapping of a specific +// interface. +func interfaceTable(ifindex int) ([]Interface, os.Error) { + return nil, nil +} + +// If the ifindex is zero, interfaceAddrTable returns addresses +// for all network interfaces. Otherwise it returns addresses +// for a specific interface. +func interfaceAddrTable(ifindex int) ([]Addr, os.Error) { + return nil, nil +} diff --git a/src/pkg/net/interface_test.go b/src/pkg/net/interface_test.go new file mode 100644 index 000000000..938434623 --- /dev/null +++ b/src/pkg/net/interface_test.go @@ -0,0 +1,90 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "bytes" + "testing" +) + +func sameInterface(i, j *Interface) bool { + if i == nil || j == nil { + return false + } + if i.Index == j.Index && i.Name == j.Name && bytes.Equal(i.HardwareAddr, j.HardwareAddr) { + return true + } + return false +} + +func interfaceFlagsString(ifi *Interface) string { + fs := "<" + if ifi.IsUp() { + fs += "UP," + } + if ifi.CanBroadcast() { + fs += "BROADCAST," + } + if ifi.IsLoopback() { + fs += "LOOPBACK," + } + if ifi.IsPointToPoint() { + fs += "POINTOPOINT," + } + if ifi.CanMulticast() { + fs += "MULTICAST," + } + if len(fs) > 1 { + fs = fs[:len(fs)-1] + } + fs += ">" + return fs +} + +func TestInterfaces(t *testing.T) { + ift, err := Interfaces() + if err != nil { + t.Fatalf("Interfaces() failed: %v", err) + } + t.Logf("table: len/cap = %v/%v\n", len(ift), cap(ift)) + + for _, ifi := range ift { + ifxi, err := InterfaceByIndex(ifi.Index) + if err != nil { + t.Fatalf("InterfaceByIndex(%#q) failed: %v", ifi.Index, err) + } + if !sameInterface(ifxi, &ifi) { + t.Fatalf("InterfaceByIndex(%#q) = %v, want %v", ifi.Index, *ifxi, ifi) + } + ifxn, err := InterfaceByName(ifi.Name) + if err != nil { + t.Fatalf("InterfaceByName(%#q) failed: %v", ifi.Name, err) + } + if !sameInterface(ifxn, &ifi) { + t.Fatalf("InterfaceByName(%#q) = %v, want %v", ifi.Name, *ifxn, ifi) + } + ifat, err := ifi.Addrs() + if err != nil { + t.Fatalf("Interface.Addrs() failed: %v", err) + } + t.Logf("%s: flags %s, ifindex %v, mtu %v\n", ifi.Name, interfaceFlagsString(&ifi), ifi.Index, ifi.MTU) + for _, ifa := range ifat { + t.Logf("\tinterface address %s\n", ifa.String()) + } + t.Logf("\thardware address %v", ifi.HardwareAddr.String()) + } +} + +func TestInterfaceAddrs(t *testing.T) { + ifat, err := InterfaceAddrs() + if err != nil { + t.Fatalf("InterfaceAddrs() failed: %v", err) + } + t.Logf("table: len/cap = %v/%v\n", len(ifat), cap(ifat)) + + for _, ifa := range ifat { + t.Logf("interface address %s\n", ifa.String()) + } +} diff --git a/src/pkg/net/ip.go b/src/pkg/net/ip.go index 61b2c687e..b0e2c4205 100644 --- a/src/pkg/net/ip.go +++ b/src/pkg/net/ip.go @@ -75,10 +75,71 @@ var ( // Well-known IPv6 addresses var ( - IPzero = make(IP, IPv6len) // all zeros - IPv6loopback = IP([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) + IPv6zero = IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + IPv6unspecified = IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + IPv6loopback = IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} + IPv6interfacelocalallnodes = IP{0xff, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x01} + IPv6linklocalallnodes = IP{0xff, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x01} + IPv6linklocalallrouters = IP{0xff, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x02} ) +// IsUnspecified returns true if ip is an unspecified address. +func (ip IP) IsUnspecified() bool { + if ip.Equal(IPv4zero) || ip.Equal(IPv6unspecified) { + return true + } + return false +} + +// IsLoopback returns true if ip is a loopback address. +func (ip IP) IsLoopback() bool { + if ip4 := ip.To4(); ip4 != nil && ip4[0] == 127 { + return true + } + return ip.Equal(IPv6loopback) +} + +// IsMulticast returns true if ip is a multicast address. +func (ip IP) IsMulticast() bool { + if ip4 := ip.To4(); ip4 != nil && ip4[0]&0xf0 == 0xe0 { + return true + } + return ip[0] == 0xff +} + +// IsInterfaceLinkLocalMulticast returns true if ip is +// an interface-local multicast address. +func (ip IP) IsInterfaceLocalMulticast() bool { + return len(ip) == IPv6len && ip[0] == 0xff && ip[1]&0x0f == 0x01 +} + +// IsLinkLocalMulticast returns true if ip is a link-local +// multicast address. +func (ip IP) IsLinkLocalMulticast() bool { + if ip4 := ip.To4(); ip4 != nil && ip4[0] == 224 && ip4[1] == 0 && ip4[2] == 0 { + return true + } + return ip[0] == 0xff && ip[1]&0x0f == 0x02 +} + +// IsLinkLocalUnicast returns true if ip is a link-local +// unicast address. +func (ip IP) IsLinkLocalUnicast() bool { + if ip4 := ip.To4(); ip4 != nil && ip4[0] == 169 && ip4[1] == 254 { + return true + } + return ip[0] == 0xfe && ip[1]&0xc0 == 0x80 +} + +// IsGlobalUnicast returns true if ip is a global unicast +// address. +func (ip IP) IsGlobalUnicast() bool { + return !ip.IsUnspecified() && + !ip.IsLoopback() && + !ip.IsMulticast() && + !ip.IsLinkLocalUnicast() +} + // Is p all zeros? func isZeros(p IP) bool { for i := 0; i < len(p); i++ { diff --git a/src/pkg/net/ip_test.go b/src/pkg/net/ip_test.go index 2008953ef..b189b10c4 100644 --- a/src/pkg/net/ip_test.go +++ b/src/pkg/net/ip_test.go @@ -9,6 +9,7 @@ import ( "reflect" "testing" "os" + "runtime" ) func isEqual(a, b []byte) bool { @@ -31,11 +32,7 @@ var parseiptests = []struct { {"abc", nil}, {"123:", nil}, {"::ffff:127.0.0.1", IPv4(127, 0, 0, 1)}, - {"2001:4860:0:2001::68", - IP{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01, - 0, 0, 0, 0, 0, 0, 0x00, 0x68, - }, - }, + {"2001:4860:0:2001::68", IP{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01, 0, 0, 0, 0, 0, 0, 0x00, 0x68}}, {"::ffff:4a7d:1363", IPv4(74, 125, 19, 99)}, } @@ -52,29 +49,21 @@ var ipstringtests = []struct { out string }{ // cf. RFC 5952 (A Recommendation for IPv6 Address Text Representation) - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, - 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1}, "2001:db8::123:12:1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0x1}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x1}, "2001:db8::1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0x1, - 0, 0, 0, 0x1, 0, 0, 0, 0x1}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1}, "2001:db8:0:1:0:1:0:1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0x1, 0, 0, - 0, 0x1, 0, 0, 0, 0x1, 0, 0}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0}, "2001:db8:1:0:1:0:1:0"}, - {IP{0x20, 0x1, 0, 0, 0, 0, 0, 0, - 0, 0x1, 0, 0, 0, 0, 0, 0x1}, + {IP{0x20, 0x1, 0, 0, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, "2001::1:0:0:1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, - 0, 0x1, 0, 0, 0, 0, 0, 0}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0}, "2001:db8:0:0:1::"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, - 0, 0x1, 0, 0, 0, 0, 0, 0x1}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, "2001:db8::1:0:0:1"}, - {IP{0x20, 0x1, 0xD, 0xB8, 0, 0, 0, 0, - 0, 0xA, 0, 0xB, 0, 0xC, 0, 0xD}, + {IP{0x20, 0x1, 0xD, 0xB8, 0, 0, 0, 0, 0, 0xA, 0, 0xB, 0, 0xC, 0, 0xD}, "2001:db8::a:b:c:d"}, } @@ -143,3 +132,84 @@ func TestJoinHostPort(t *testing.T) { } } } + +var ipaftests = []struct { + in IP + af4 bool + af6 bool +}{ + {IPv4bcast, true, false}, + {IPv4allsys, true, false}, + {IPv4allrouter, true, false}, + {IPv4zero, true, false}, + {IPv4(224, 0, 0, 1), true, false}, + {IPv4(127, 0, 0, 1), true, false}, + {IPv4(240, 0, 0, 1), true, false}, + {IPv6unspecified, false, true}, + {IPv6loopback, false, true}, + {IPv6interfacelocalallnodes, false, true}, + {IPv6linklocalallnodes, false, true}, + {IPv6linklocalallrouters, false, true}, + {ParseIP("ff05::a:b:c:d"), false, true}, + {ParseIP("fe80::1:2:3:4"), false, true}, + {ParseIP("2001:db8::123:12:1"), false, true}, +} + +func TestIPAddrFamily(t *testing.T) { + for _, tt := range ipaftests { + if af := tt.in.To4() != nil; af != tt.af4 { + t.Errorf("verifying IPv4 address family for %#q = %v, want %v", tt.in, af, tt.af4) + } + if af := len(tt.in) == IPv6len && tt.in.To4() == nil; af != tt.af6 { + t.Errorf("verifying IPv6 address family for %#q = %v, want %v", tt.in, af, tt.af6) + } + } +} + +var ipscopetests = []struct { + scope func(IP) bool + in IP + ok bool +}{ + {IP.IsUnspecified, IPv4zero, true}, + {IP.IsUnspecified, IPv4(127, 0, 0, 1), false}, + {IP.IsUnspecified, IPv6unspecified, true}, + {IP.IsUnspecified, IPv6interfacelocalallnodes, false}, + {IP.IsLoopback, IPv4(127, 0, 0, 1), true}, + {IP.IsLoopback, IPv4(127, 255, 255, 254), true}, + {IP.IsLoopback, IPv4(128, 1, 2, 3), false}, + {IP.IsLoopback, IPv6loopback, true}, + {IP.IsLoopback, IPv6linklocalallrouters, false}, + {IP.IsMulticast, IPv4(224, 0, 0, 0), true}, + {IP.IsMulticast, IPv4(239, 0, 0, 0), true}, + {IP.IsMulticast, IPv4(240, 0, 0, 0), false}, + {IP.IsMulticast, IPv6linklocalallnodes, true}, + {IP.IsMulticast, IP{0xff, 0x05, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true}, + {IP.IsMulticast, IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false}, + {IP.IsLinkLocalMulticast, IPv4(224, 0, 0, 0), true}, + {IP.IsLinkLocalMulticast, IPv4(239, 0, 0, 0), false}, + {IP.IsLinkLocalMulticast, IPv6linklocalallrouters, true}, + {IP.IsLinkLocalMulticast, IP{0xff, 0x05, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false}, + {IP.IsLinkLocalUnicast, IPv4(169, 254, 0, 0), true}, + {IP.IsLinkLocalUnicast, IPv4(169, 255, 0, 0), false}, + {IP.IsLinkLocalUnicast, IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true}, + {IP.IsLinkLocalUnicast, IP{0xfe, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false}, + {IP.IsGlobalUnicast, IPv4(240, 0, 0, 0), true}, + {IP.IsGlobalUnicast, IPv4(232, 0, 0, 0), false}, + {IP.IsGlobalUnicast, IPv4(169, 254, 0, 0), false}, + {IP.IsGlobalUnicast, IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1}, true}, + {IP.IsGlobalUnicast, IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false}, + {IP.IsGlobalUnicast, IP{0xff, 0x05, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false}, +} + +func name(f interface{}) string { + return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() +} + +func TestIPAddrScope(t *testing.T) { + for _, tt := range ipscopetests { + if ok := tt.scope(tt.in); ok != tt.ok { + t.Errorf("%s(%#q) = %v, want %v", name(tt.scope), tt.in, ok, tt.ok) + } + } +} diff --git a/src/pkg/net/ipraw_test.go b/src/pkg/net/ipraw_test.go index ee8c71fc1..7cc9604b5 100644 --- a/src/pkg/net/ipraw_test.go +++ b/src/pkg/net/ipraw_test.go @@ -60,7 +60,8 @@ func parsePingReply(p []byte) (id, seq int) { } var srchost = flag.String("srchost", "", "Source of the ICMP ECHO request") -var dsthost = flag.String("dsthost", "localhost", "Destination for the ICMP ECHO request") +// 127.0.0.1 because this is an IPv4-specific test. +var dsthost = flag.String("dsthost", "127.0.0.1", "Destination for the ICMP ECHO request") // test (raw) IP socket using ICMP func TestICMP(t *testing.T) { @@ -74,15 +75,15 @@ func TestICMP(t *testing.T) { err os.Error ) if *srchost != "" { - laddr, err = ResolveIPAddr(*srchost) + laddr, err = ResolveIPAddr("ip4", *srchost) if err != nil { - t.Fatalf(`net.ResolveIPAddr("%v") = %v, %v`, *srchost, laddr, err) + t.Fatalf(`net.ResolveIPAddr("ip4", %v") = %v, %v`, *srchost, laddr, err) } } - raddr, err := ResolveIPAddr(*dsthost) + raddr, err := ResolveIPAddr("ip4", *dsthost) if err != nil { - t.Fatalf(`net.ResolveIPAddr("%v") = %v, %v`, *dsthost, raddr, err) + t.Fatalf(`net.ResolveIPAddr("ip4", %v") = %v, %v`, *dsthost, raddr, err) } c, err := ListenIP("ip4:icmp", laddr) diff --git a/src/pkg/net/iprawsock.go b/src/pkg/net/iprawsock.go index 5be6fe4e0..a811027b1 100644 --- a/src/pkg/net/iprawsock.go +++ b/src/pkg/net/iprawsock.go @@ -43,7 +43,7 @@ func (a *IPAddr) family() int { if a == nil || len(a.IP) <= 4 { return syscall.AF_INET } - if ip := a.IP.To4(); ip != nil { + if a.IP.To4() != nil { return syscall.AF_INET } return syscall.AF_INET6 @@ -61,10 +61,11 @@ func (a *IPAddr) toAddr() sockaddr { } // ResolveIPAddr parses addr as a IP address and resolves domain -// names to numeric addresses. A literal IPv6 host address must be +// names to numeric addresses on the network net, which must be +// "ip", "ip4" or "ip6". A literal IPv6 host address must be // enclosed in square brackets, as in "[::]". -func ResolveIPAddr(addr string) (*IPAddr, os.Error) { - ip, err := hostToIP(addr) +func ResolveIPAddr(net, addr string) (*IPAddr, os.Error) { + ip, err := hostToIP(net, addr) if err != nil { return nil, err } @@ -234,32 +235,36 @@ func (c *IPConn) WriteTo(b []byte, addr Addr) (n int, err os.Error) { } // Convert "host" into IP address. -func hostToIP(host string) (ip IP, err os.Error) { +func hostToIP(net, host string) (ip IP, err os.Error) { var addr IP // Try as an IP address. addr = ParseIP(host) if addr == nil { + filter := anyaddr + if net != "" && net[len(net)-1] == '4' { + filter = ipv4only + } + if net != "" && net[len(net)-1] == '6' { + filter = ipv6only + } // Not an IP address. Try as a DNS name. addrs, err1 := LookupHost(host) if err1 != nil { err = err1 goto Error } - addr = firstSupportedAddr(anyaddr, addrs) + addr = firstFavoriteAddr(filter, addrs) if addr == nil { // should not happen - err = &AddrError{"LookupHost returned invalid address", addrs[0]} + err = &AddrError{"LookupHost returned no suitable address", addrs[0]} goto Error } } - return addr, nil - Error: return nil, err } - var protocols map[string]int func readProtocols() { @@ -285,7 +290,7 @@ func readProtocols() { } } -func netProtoSplit(netProto string) (net string, proto int, err os.Error) { +func splitNetProto(netProto string) (net string, proto int, err os.Error) { onceReadProtocols.Do(readProtocols) i := last(netProto, ':') if i < 0 { // no colon @@ -307,7 +312,7 @@ func netProtoSplit(netProto string) (net string, proto int, err os.Error) { // DialIP connects to the remote address raddr on the network net, // which must be "ip", "ip4", or "ip6". func DialIP(netProto string, laddr, raddr *IPAddr) (c *IPConn, err os.Error) { - net, proto, err := netProtoSplit(netProto) + net, proto, err := splitNetProto(netProto) if err != nil { return } @@ -331,7 +336,7 @@ func DialIP(netProto string, laddr, raddr *IPAddr) (c *IPConn, err os.Error) { // and WriteTo methods can be used to receive and send IP // packets with per-packet addressing. func ListenIP(netProto string, laddr *IPAddr) (c *IPConn, err os.Error) { - net, proto, err := netProtoSplit(netProto) + net, proto, err := splitNetProto(netProto) if err != nil { return } diff --git a/src/pkg/net/ipsock.go b/src/pkg/net/ipsock.go index e8bcac646..0b8c388f1 100644 --- a/src/pkg/net/ipsock.go +++ b/src/pkg/net/ipsock.go @@ -15,25 +15,105 @@ import ( // only dealing with IPv4 sockets? As long as the host system // understands IPv6, it's okay to pass IPv4 addresses to the IPv6 // interface. That simplifies our code and is most general. -// Unfortunately, we need to run on kernels built without IPv6 support too. -// So probe the kernel to figure it out. -func kernelSupportsIPv6() bool { - s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) - if err != 0 { - return false +// Unfortunately, we need to run on kernels built without IPv6 +// support too. So probe the kernel to figure it out. +// +// probeIPv6Stack probes both basic IPv6 capability and IPv6 IPv4- +// mapping capability which is controlled by IPV6_V6ONLY socket +// option and/or kernel state "net.inet6.ip6.v6only". +// It returns two boolean values. If the first boolean value is +// true, kernel supports basic IPv6 functionality. If the second +// boolean value is true, kernel supports IPv6 IPv4-mapping. +func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { + var probes = []struct { + s int + la TCPAddr + ok bool + }{ + // IPv6 communication capability + {-1, TCPAddr{IP: ParseIP("::1")}, false}, + // IPv6 IPv4-mapped address communication capability + {-1, TCPAddr{IP: IPv4(127, 0, 0, 1)}, false}, } - defer closesocket(s) + var errno int - la := &TCPAddr{IP: IPv4(127, 0, 0, 1)} - sa, oserr := la.toAddr().sockaddr(syscall.AF_INET6) - if oserr != nil { - return false + for i := range probes { + probes[i].s, errno = syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + if errno != 0 { + continue + } + defer closesocket(probes[i].s) + sa, err := probes[i].la.toAddr().sockaddr(syscall.AF_INET6) + if err != nil { + continue + } + errno = syscall.Bind(probes[i].s, sa) + if errno != 0 { + continue + } + probes[i].ok = true } - return syscall.Bind(s, sa) == 0 + return probes[0].ok, probes[1].ok } -var preferIPv4 = !kernelSupportsIPv6() +var supportsIPv6, supportsIPv4map = probeIPv6Stack() + +// favoriteAddrFamily returns the appropriate address family to +// the given net, raddr, laddr and mode. At first it figures +// address family out from the net. If mode indicates "listen" +// and laddr.(type).IP is nil, it assumes that the user wants to +// make a passive connection with wildcard address family, both +// INET and INET6, and wildcard address. Otherwise guess: if the +// addresses are IPv4 then returns INET, or else returns INET6. +func favoriteAddrFamily(net string, raddr, laddr sockaddr, mode string) int { + switch net[len(net)-1] { + case '4': + return syscall.AF_INET + case '6': + return syscall.AF_INET6 + } + + if mode == "listen" { + switch a := laddr.(type) { + case *TCPAddr: + if a.IP == nil && supportsIPv6 { + return syscall.AF_INET6 + } + case *UDPAddr: + if a.IP == nil && supportsIPv6 { + return syscall.AF_INET6 + } + case *IPAddr: + if a.IP == nil && supportsIPv6 { + return syscall.AF_INET6 + } + } + } + + if (laddr == nil || laddr.family() == syscall.AF_INET) && + (raddr == nil || raddr.family() == syscall.AF_INET) { + return syscall.AF_INET + } + return syscall.AF_INET6 +} + +func firstFavoriteAddr(filter func(IP) IP, addrs []string) (addr IP) { + if filter == anyaddr { + // We'll take any IP address, but since the dialing code + // does not yet try multiple addresses, prefer to use + // an IPv4 address if possible. This is especially relevant + // if localhost resolves to [ipv6-localhost, ipv4-localhost]. + // Too much code assumes localhost == ipv4-localhost. + addr = firstSupportedAddr(ipv4only, addrs) + if addr == nil { + addr = firstSupportedAddr(anyaddr, addrs) + } + } else { + addr = firstSupportedAddr(filter, addrs) + } + return +} func firstSupportedAddr(filter func(IP) IP, addrs []string) IP { for _, s := range addrs { @@ -44,22 +124,28 @@ func firstSupportedAddr(filter func(IP) IP, addrs []string) IP { return nil } -func anyaddr(x IP) IP { return x } +func anyaddr(x IP) IP { + if x4 := x.To4(); x4 != nil { + return x4 + } + if supportsIPv6 { + return x + } + return nil +} + func ipv4only(x IP) IP { return x.To4() } func ipv6only(x IP) IP { // Only return addresses that we can use // with the kernel's IPv6 addressing modes. - // If preferIPv4 is set, it means the IPv6 stack - // cannot take IPv4 addresses directly (we prefer - // to use the IPv4 stack) so reject IPv4 addresses. - if x.To4() != nil && preferIPv4 { - return nil + if len(x) == IPv6len && x.To4() == nil && supportsIPv6 { + return x } - return x + return nil } -// TODO(rsc): if syscall.OS == "linux", we're supposd to read +// TODO(rsc): if syscall.OS == "linux", we're supposed to read // /proc/sys/net/core/somaxconn, // to take advantage of kernels that have raised the limit. func listenBacklog() int { return syscall.SOMAXCONN } @@ -75,26 +161,9 @@ type sockaddr interface { } func internetSocket(net string, laddr, raddr sockaddr, socktype, proto int, mode string, toAddr func(syscall.Sockaddr) Addr) (fd *netFD, err os.Error) { - // Figure out IP version. - // If network has a suffix like "tcp4", obey it. var oserr os.Error - family := syscall.AF_INET6 - switch net[len(net)-1] { - case '4': - family = syscall.AF_INET - case '6': - // nothing to do - default: - // Otherwise, guess. - // If the addresses are IPv4 and we prefer IPv4, use 4; else 6. - if preferIPv4 && - (laddr == nil || laddr.family() == syscall.AF_INET) && - (raddr == nil || raddr.family() == syscall.AF_INET) { - family = syscall.AF_INET - } - } - var la, ra syscall.Sockaddr + family := favoriteAddrFamily(net, raddr, laddr, mode) if laddr != nil { if la, oserr = laddr.sockaddr(family); oserr != nil { goto Error @@ -119,25 +188,6 @@ Error: return nil, &OpError{mode, net, addr, oserr} } -func getip(fd int, remote bool) (ip []byte, port int, ok bool) { - // No attempt at error reporting because - // there are no possible errors, and the - // caller won't report them anyway. - var sa syscall.Sockaddr - if remote { - sa, _ = syscall.Getpeername(fd) - } else { - sa, _ = syscall.Getsockname(fd) - } - switch sa := sa.(type) { - case *syscall.SockaddrInet4: - return sa.Addr[0:], sa.Port, true - case *syscall.SockaddrInet6: - return sa.Addr[0:], sa.Port, true - } - return -} - type InvalidAddrError string func (e InvalidAddrError) String() string { return string(e) } @@ -161,13 +211,13 @@ func ipToSockaddr(family int, ip IP, port int) (syscall.Sockaddr, os.Error) { return s, nil case syscall.AF_INET6: if len(ip) == 0 { - ip = IPzero + ip = IPv6zero } // IPv4 callers use 0.0.0.0 to mean "announce on any available address". // In IPv6 mode, Linux treats that as meaning "announce on 0.0.0.0", // which it refuses to do. Rewrite to the IPv6 all zeros. - if p4 := ip.To4(); p4 != nil && p4[0] == 0 && p4[1] == 0 && p4[2] == 0 && p4[3] == 0 { - ip = IPzero + if ip.Equal(IPv4zero) { + ip = IPv6zero } if ip = ip.To16(); ip == nil { return nil, InvalidAddrError("non-IPv6 address") @@ -231,9 +281,10 @@ func hostPortToIP(net, hostport string) (ip IP, iport int, err os.Error) { addr = ParseIP(host) if addr == nil { filter := anyaddr - if len(net) >= 4 && net[3] == '4' { + if net != "" && net[len(net)-1] == '4' { filter = ipv4only - } else if len(net) >= 4 && net[3] == '6' { + } + if net != "" && net[len(net)-1] == '6' { filter = ipv6only } // Not an IP address. Try as a DNS name. @@ -242,22 +293,10 @@ func hostPortToIP(net, hostport string) (ip IP, iport int, err os.Error) { err = err1 goto Error } - if filter == anyaddr { - // We'll take any IP address, but since the dialing code - // does not yet try multiple addresses, prefer to use - // an IPv4 address if possible. This is especially relevant - // if localhost resolves to [ipv6-localhost, ipv4-localhost]. - // Too much code assumes localhost == ipv4-localhost. - addr = firstSupportedAddr(ipv4only, addrs) - if addr == nil { - addr = firstSupportedAddr(anyaddr, addrs) - } - } else { - addr = firstSupportedAddr(filter, addrs) - } + addr = firstFavoriteAddr(filter, addrs) if addr == nil { // should not happen - err = &AddrError{"LookupHost returned invalid address", addrs[0]} + err = &AddrError{"LookupHost returned no suitable address", addrs[0]} goto Error } } diff --git a/src/pkg/net/resolv_windows.go b/src/pkg/net/resolv_windows.go index 3506ea177..f7c3f51be 100644 --- a/src/pkg/net/resolv_windows.go +++ b/src/pkg/net/resolv_windows.go @@ -113,6 +113,10 @@ func reverseaddr(addr string) (arpa string, err os.Error) { panic("unimplemented") } +func answer(name, server string, dns *dnsMsg, qtype uint16) (cname string, addrs []dnsRR, err os.Error) { + panic("unimplemented") +} + // DNSError represents a DNS lookup error. type DNSError struct { Error string // description of the error diff --git a/src/pkg/net/sendfile_linux.go b/src/pkg/net/sendfile_linux.go new file mode 100644 index 000000000..6a5a06c8c --- /dev/null +++ b/src/pkg/net/sendfile_linux.go @@ -0,0 +1,84 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "io" + "os" + "syscall" +) + +// maxSendfileSize is the largest chunk size we ask the kernel to copy +// at a time. +const maxSendfileSize int = 4 << 20 + +// sendFile copies the contents of r to c using the sendfile +// system call to minimize copies. +// +// if handled == true, sendFile returns the number of bytes copied and any +// non-EOF error. +// +// if handled == false, sendFile performed no work. +func sendFile(c *netFD, r io.Reader) (written int64, err os.Error, handled bool) { + var remain int64 = 1 << 62 // by default, copy until EOF + + lr, ok := r.(*io.LimitedReader) + if ok { + remain, r = lr.N, lr.R + if remain <= 0 { + return 0, nil, true + } + } + f, ok := r.(*os.File) + if !ok { + return 0, nil, false + } + + c.wio.Lock() + defer c.wio.Unlock() + c.incref() + defer c.decref() + if c.wdeadline_delta > 0 { + // This is a little odd that we're setting the timeout + // for the entire file but Write has the same issue + // (if one slurps the whole file into memory and + // do one large Write). At least they're consistent. + c.wdeadline = pollserver.Now() + c.wdeadline_delta + } else { + c.wdeadline = 0 + } + + dst := c.sysfd + src := f.Fd() + for remain > 0 { + n := maxSendfileSize + if int64(n) > remain { + n = int(remain) + } + n, errno := syscall.Sendfile(dst, src, nil, n) + if n > 0 { + written += int64(n) + remain -= int64(n) + } + if n == 0 && errno == 0 { + break + } + if errno == syscall.EAGAIN && c.wdeadline >= 0 { + pollserver.WaitWrite(c) + continue + } + if errno != 0 { + // This includes syscall.ENOSYS (no kernel + // support) and syscall.EINVAL (fd types which + // don't implement sendfile together) + err = &OpError{"sendfile", c.net, c.raddr, os.Errno(errno)} + break + } + } + if lr != nil { + lr.N = remain + } + return written, err, written > 0 +} diff --git a/src/pkg/net/sendfile_stub.go b/src/pkg/net/sendfile_stub.go new file mode 100644 index 000000000..43e8104e9 --- /dev/null +++ b/src/pkg/net/sendfile_stub.go @@ -0,0 +1,14 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "io" + "os" +) + +func sendFile(c *netFD, r io.Reader) (n int64, err os.Error, handled bool) { + return 0, nil, false +} diff --git a/src/pkg/net/server_test.go b/src/pkg/net/server_test.go index 075748b83..36780d789 100644 --- a/src/pkg/net/server_test.go +++ b/src/pkg/net/server_test.go @@ -92,15 +92,22 @@ func connect(t *testing.T, network, addr string, isEmpty bool) { } func doTest(t *testing.T, network, listenaddr, dialaddr string) { - t.Logf("Test %s %s %s\n", network, listenaddr, dialaddr) + t.Logf("Test %q %q %q\n", network, listenaddr, dialaddr) + switch listenaddr { + case "", "0.0.0.0", "[::]", "[::ffff:0.0.0.0]": + if testing.Short() || avoidMacFirewall { + t.Logf("skip wildcard listen during short test") + return + } + } listening := make(chan string) done := make(chan int) - if network == "tcp" { + if network == "tcp" || network == "tcp4" || network == "tcp6" { listenaddr += ":0" // any available port } go runServe(t, network, listenaddr, listening, done) addr := <-listening // wait for server to start - if network == "tcp" { + if network == "tcp" || network == "tcp4" || network == "tcp6" { dialaddr += addr[strings.LastIndex(addr, ":"):] } connect(t, network, dialaddr, false) @@ -108,10 +115,33 @@ func doTest(t *testing.T, network, listenaddr, dialaddr string) { } func TestTCPServer(t *testing.T) { + doTest(t, "tcp", "", "127.0.0.1") + doTest(t, "tcp", "0.0.0.0", "127.0.0.1") doTest(t, "tcp", "127.0.0.1", "127.0.0.1") - if kernelSupportsIPv6() { + doTest(t, "tcp4", "", "127.0.0.1") + doTest(t, "tcp4", "0.0.0.0", "127.0.0.1") + doTest(t, "tcp4", "127.0.0.1", "127.0.0.1") + if supportsIPv6 { + doTest(t, "tcp", "", "[::1]") + doTest(t, "tcp", "[::]", "[::1]") doTest(t, "tcp", "[::1]", "[::1]") + doTest(t, "tcp6", "", "[::1]") + doTest(t, "tcp6", "[::]", "[::1]") + doTest(t, "tcp6", "[::1]", "[::1]") + } + if supportsIPv6 && supportsIPv4map { + doTest(t, "tcp", "[::ffff:0.0.0.0]", "127.0.0.1") + doTest(t, "tcp", "[::]", "127.0.0.1") + doTest(t, "tcp4", "[::ffff:0.0.0.0]", "127.0.0.1") + doTest(t, "tcp6", "", "127.0.0.1") + doTest(t, "tcp6", "[::ffff:0.0.0.0]", "127.0.0.1") + doTest(t, "tcp6", "[::]", "127.0.0.1") doTest(t, "tcp", "127.0.0.1", "[::ffff:127.0.0.1]") + doTest(t, "tcp", "[::ffff:127.0.0.1]", "127.0.0.1") + doTest(t, "tcp4", "127.0.0.1", "[::ffff:127.0.0.1]") + doTest(t, "tcp4", "[::ffff:127.0.0.1]", "127.0.0.1") + doTest(t, "tcp6", "127.0.0.1", "[::ffff:127.0.0.1]") + doTest(t, "tcp6", "[::ffff:127.0.0.1]", "127.0.0.1") } } @@ -186,7 +216,7 @@ func TestUDPServer(t *testing.T) { for _, isEmpty := range []bool{false, true} { doTestPacket(t, "udp", "0.0.0.0", "127.0.0.1", isEmpty) doTestPacket(t, "udp", "", "127.0.0.1", isEmpty) - if kernelSupportsIPv6() { + if supportsIPv6 && supportsIPv4map { doTestPacket(t, "udp", "[::]", "[::ffff:127.0.0.1]", isEmpty) doTestPacket(t, "udp", "[::]", "127.0.0.1", isEmpty) doTestPacket(t, "udp", "0.0.0.0", "[::ffff:127.0.0.1]", isEmpty) diff --git a/src/pkg/net/sock.go b/src/pkg/net/sock.go index bd88f7ece..eae7f3711 100644 --- a/src/pkg/net/sock.go +++ b/src/pkg/net/sock.go @@ -7,6 +7,7 @@ package net import ( + "io" "os" "reflect" "syscall" @@ -32,17 +33,7 @@ func socket(net string, f, p, t int, la, ra syscall.Sockaddr, toAddr func(syscal syscall.CloseOnExec(s) syscall.ForkLock.RUnlock() - // Allow reuse of recently-used addresses. - syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1) - - // Allow broadcast. - syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1) - - if f == syscall.AF_INET6 { - // using ip, tcp, udp, etc. - // allow both protocols even if the OS default is otherwise. - syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 0) - } + setKernelSpecificSockopt(s, f) if la != nil { e = syscall.Bind(s, la) @@ -164,15 +155,13 @@ func (e *UnknownSocketError) String() string { return "unknown socket address type " + reflect.TypeOf(e.sa).String() } -func sockaddrToString(sa syscall.Sockaddr) (name string, err os.Error) { - switch a := sa.(type) { - case *syscall.SockaddrInet4: - return JoinHostPort(IP(a.Addr[0:]).String(), itoa(a.Port)), nil - case *syscall.SockaddrInet6: - return JoinHostPort(IP(a.Addr[0:]).String(), itoa(a.Port)), nil - case *syscall.SockaddrUnix: - return a.Name, nil - } +type writerOnly struct { + io.Writer +} - return "", &UnknownSocketError{sa} +// Fallback implementation of io.ReaderFrom's ReadFrom, when sendfile isn't +// applicable. +func genericReadFrom(w io.Writer, r io.Reader) (n int64, err os.Error) { + // Use wrapper to hide existing r.ReadFrom from io.Copy. + return io.Copy(writerOnly{w}, r) } diff --git a/src/pkg/net/sock_bsd.go b/src/pkg/net/sock_bsd.go new file mode 100644 index 000000000..5fd52074a --- /dev/null +++ b/src/pkg/net/sock_bsd.go @@ -0,0 +1,31 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Sockets for BSD variants + +package net + +import ( + "syscall" +) + +func setKernelSpecificSockopt(s, f int) { + // Allow reuse of recently-used addresses. + syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1) + + // Allow reuse of recently-used ports. + // This option is supported only in descendants of 4.4BSD, + // to make an effective multicast application and an application + // that requires quick draw possible. + syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEPORT, 1) + + // Allow broadcast. + syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1) + + if f == syscall.AF_INET6 { + // using ip, tcp, udp, etc. + // allow both protocols even if the OS default is otherwise. + syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 0) + } +} diff --git a/src/pkg/net/sock_linux.go b/src/pkg/net/sock_linux.go new file mode 100644 index 000000000..ec31e803b --- /dev/null +++ b/src/pkg/net/sock_linux.go @@ -0,0 +1,25 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Sockets for Linux + +package net + +import ( + "syscall" +) + +func setKernelSpecificSockopt(s, f int) { + // Allow reuse of recently-used addresses. + syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1) + + // Allow broadcast. + syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1) + + if f == syscall.AF_INET6 { + // using ip, tcp, udp, etc. + // allow both protocols even if the OS default is otherwise. + syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 0) + } +} diff --git a/src/pkg/net/sock_windows.go b/src/pkg/net/sock_windows.go new file mode 100644 index 000000000..e17c60b98 --- /dev/null +++ b/src/pkg/net/sock_windows.go @@ -0,0 +1,25 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Sockets for Windows + +package net + +import ( + "syscall" +) + +func setKernelSpecificSockopt(s, f int) { + // Allow reuse of recently-used addresses and ports. + syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1) + + // Allow broadcast. + syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1) + + if f == syscall.AF_INET6 { + // using ip, tcp, udp, etc. + // allow both protocols even if the OS default is otherwise. + syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 0) + } +} diff --git a/src/pkg/net/tcpsock.go b/src/pkg/net/tcpsock.go index d9aa7cf19..9ee6c14f7 100644 --- a/src/pkg/net/tcpsock.go +++ b/src/pkg/net/tcpsock.go @@ -7,6 +7,7 @@ package net import ( + "io" "os" "syscall" ) @@ -41,7 +42,7 @@ func (a *TCPAddr) family() int { if a == nil || len(a.IP) <= 4 { return syscall.AF_INET } - if ip := a.IP.To4(); ip != nil { + if a.IP.To4() != nil { return syscall.AF_INET } return syscall.AF_INET6 @@ -60,10 +61,11 @@ func (a *TCPAddr) toAddr() sockaddr { // ResolveTCPAddr parses addr as a TCP address of the form // host:port and resolves domain names or port names to -// numeric addresses. A literal IPv6 host address must be +// numeric addresses on the network net, which must be "tcp", +// "tcp4" or "tcp6". A literal IPv6 host address must be // enclosed in square brackets, as in "[::]:80". -func ResolveTCPAddr(network, addr string) (*TCPAddr, os.Error) { - ip, port, err := hostPortToIP(network, addr) +func ResolveTCPAddr(net, addr string) (*TCPAddr, os.Error) { + ip, port, err := hostPortToIP(net, addr) if err != nil { return nil, err } @@ -94,6 +96,14 @@ func (c *TCPConn) Read(b []byte) (n int, err os.Error) { return c.fd.Read(b) } +// ReadFrom implements the io.ReaderFrom ReadFrom method. +func (c *TCPConn) ReadFrom(r io.Reader) (int64, os.Error) { + if n, err, handled := sendFile(c.fd, r); handled { + return n, err + } + return genericReadFrom(c, r) +} + // Write implements the net.Conn Write method. func (c *TCPConn) Write(b []byte) (n int, err os.Error) { if !c.ok() { diff --git a/src/pkg/net/textproto/reader.go b/src/pkg/net/textproto/reader.go index ac1278689..e65374903 100644 --- a/src/pkg/net/textproto/reader.go +++ b/src/pkg/net/textproto/reader.go @@ -237,7 +237,7 @@ func (r *Reader) ReadResponse(expectCode int) (code int, message string, err os. // to a method on r. // // Dot encoding is a common framing used for data blocks -// in text protcols like SMTP. The data consists of a sequence +// in text protocols such as SMTP. The data consists of a sequence // of lines, each of which ends in "\r\n". The sequence itself // ends at a line containing just a dot: ".\r\n". Lines beginning // with a dot are escaped with an additional dot to avoid diff --git a/src/pkg/net/udpsock.go b/src/pkg/net/udpsock.go index 67684471b..5469acffa 100644 --- a/src/pkg/net/udpsock.go +++ b/src/pkg/net/udpsock.go @@ -41,7 +41,7 @@ func (a *UDPAddr) family() int { if a == nil || len(a.IP) <= 4 { return syscall.AF_INET } - if ip := a.IP.To4(); ip != nil { + if a.IP.To4() != nil { return syscall.AF_INET } return syscall.AF_INET6 @@ -60,10 +60,11 @@ func (a *UDPAddr) toAddr() sockaddr { // ResolveUDPAddr parses addr as a UDP address of the form // host:port and resolves domain names or port names to -// numeric addresses. A literal IPv6 host address must be +// numeric addresses on the network net, which must be "udp", +// "udp4" or "udp6". A literal IPv6 host address must be // enclosed in square brackets, as in "[::]:80". -func ResolveUDPAddr(network, addr string) (*UDPAddr, os.Error) { - ip, port, err := hostPortToIP(network, addr) +func ResolveUDPAddr(net, addr string) (*UDPAddr, os.Error) { + ip, port, err := hostPortToIP(net, addr) if err != nil { return nil, err } @@ -292,10 +293,10 @@ func (c *UDPConn) JoinGroup(addr IP) os.Error { if ip == nil { return &OpError{"joingroup", "udp", &IPAddr{ip}, errInvalidMulticast} } - mreq := &syscall.IpMreq{ + mreq := &syscall.IPMreq{ Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}, } - err := os.NewSyscallError("setsockopt", syscall.SetsockoptIpMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq)) + err := os.NewSyscallError("setsockopt", syscall.SetsockoptIPMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq)) if err != nil { return &OpError{"joingroup", "udp", &IPAddr{ip}, err} } @@ -311,10 +312,10 @@ func (c *UDPConn) LeaveGroup(addr IP) os.Error { if ip == nil { return &OpError{"leavegroup", "udp", &IPAddr{ip}, errInvalidMulticast} } - mreq := &syscall.IpMreq{ + mreq := &syscall.IPMreq{ Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}, } - err := os.NewSyscallError("setsockopt", syscall.SetsockoptIpMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_DROP_MEMBERSHIP, mreq)) + err := os.NewSyscallError("setsockopt", syscall.SetsockoptIPMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_DROP_MEMBERSHIP, mreq)) if err != nil { return &OpError{"leavegroup", "udp", &IPAddr{ip}, err} } diff --git a/src/pkg/netchan/import.go b/src/pkg/netchan/import.go index 0a700ca2b..7d96228c4 100644 --- a/src/pkg/netchan/import.go +++ b/src/pkg/netchan/import.go @@ -11,6 +11,7 @@ import ( "os" "reflect" "sync" + "time" ) // Import @@ -31,6 +32,9 @@ type Importer struct { chans map[int]*netChan errors chan os.Error maxId int + mu sync.Mutex // protects remaining fields + unacked int64 // number of unacknowledged sends. + seqLock sync.Mutex // guarantees messages are in sequence, only locked under mu } // NewImporter creates a new Importer object to import a set of channels @@ -42,6 +46,7 @@ func NewImporter(conn io.ReadWriter) *Importer { imp.chans = make(map[int]*netChan) imp.names = make(map[string]*netChan) imp.errors = make(chan os.Error, 10) + imp.unacked = 0 go imp.run() return imp } @@ -80,8 +85,10 @@ func (imp *Importer) run() { for { *hdr = header{} if e := imp.decode(hdrValue); e != nil { - impLog("header:", e) - imp.shutdown() + if e != os.EOF { + impLog("header:", e) + imp.shutdown() + } return } switch hdr.PayloadType { @@ -114,6 +121,9 @@ func (imp *Importer) run() { nch := imp.getChan(hdr.Id, true) if nch != nil { nch.acked() + imp.mu.Lock() + imp.unacked-- + imp.mu.Unlock() } continue default: @@ -220,10 +230,17 @@ func (imp *Importer) ImportNValues(name string, chT interface{}, dir Dir, size, } return } + // We hold the lock during transmission to guarantee messages are + // sent in order. + imp.mu.Lock() + imp.unacked++ + imp.seqLock.Lock() + imp.mu.Unlock() if err = imp.encode(hdr, payData, val.Interface()); err != nil { impLog("error encoding client send:", err) return } + imp.seqLock.Unlock() } }() } @@ -244,3 +261,27 @@ func (imp *Importer) Hangup(name string) os.Error { nc.close() return nil } + +func (imp *Importer) unackedCount() int64 { + imp.mu.Lock() + n := imp.unacked + imp.mu.Unlock() + return n +} + +// Drain waits until all messages sent from this exporter/importer, including +// those not yet sent to any server and possibly including those sent while +// Drain was executing, have been received by the exporter. In short, it +// waits until all the importer's messages have been received. +// If the timeout (measured in nanoseconds) is positive and Drain takes +// longer than that to complete, an error is returned. +func (imp *Importer) Drain(timeout int64) os.Error { + startTime := time.Nanoseconds() + for imp.unackedCount() > 0 { + if timeout > 0 && time.Nanoseconds()-startTime >= timeout { + return os.ErrorString("timeout") + } + time.Sleep(100 * 1e6) + } + return nil +} diff --git a/src/pkg/netchan/netchan_test.go b/src/pkg/netchan/netchan_test.go index fd4d8f780..8c0f9a6e4 100644 --- a/src/pkg/netchan/netchan_test.go +++ b/src/pkg/netchan/netchan_test.go @@ -178,6 +178,16 @@ func TestExportDrain(t *testing.T) { <-done } +// Not a great test but it does at least invoke Drain. +func TestImportDrain(t *testing.T) { + exp, imp := pair(t) + expDone := make(chan bool) + go exportReceive(exp, t, expDone) + <-expDone + importSend(imp, closeCount, t, nil) + imp.Drain(0) +} + // Not a great test but it does at least invoke Sync. func TestExportSync(t *testing.T) { exp, imp := pair(t) diff --git a/src/pkg/os/Makefile b/src/pkg/os/Makefile index cd9284079..497e5a958 100644 --- a/src/pkg/os/Makefile +++ b/src/pkg/os/Makefile @@ -23,9 +23,11 @@ GOFILES_freebsd=\ env_unix.go\ file_posix.go\ file_unix.go\ + path_unix.go\ sys_bsd.go\ exec_posix.go\ exec_unix.go\ + signal_unix.go\ GOFILES_darwin=\ dir_unix.go\ @@ -33,9 +35,11 @@ GOFILES_darwin=\ env_unix.go\ file_posix.go\ file_unix.go\ + path_unix.go\ sys_bsd.go\ exec_posix.go\ exec_unix.go\ + signal_unix.go\ GOFILES_linux=\ dir_unix.go\ @@ -43,9 +47,11 @@ GOFILES_linux=\ env_unix.go\ file_posix.go\ file_unix.go\ + path_unix.go\ sys_linux.go\ exec_posix.go\ exec_unix.go\ + signal_unix.go\ GOFILES_windows=\ dir_windows.go\ @@ -53,18 +59,29 @@ GOFILES_windows=\ env_windows.go\ file_posix.go\ file_windows.go\ + path_windows.go\ sys_windows.go\ exec_posix.go\ exec_windows.go\ + signal_windows.go\ GOFILES_plan9=\ dir_plan9.go\ error_plan9.go\ env_plan9.go\ file_plan9.go\ + path_plan9.go\ sys_plan9.go\ exec_plan9.go\ GOFILES+=$(GOFILES_$(GOOS)) +CLEANFILES+=signal_unix.go signal_windows.go + include ../../Make.pkg + +signal_unix.go: ../syscall/zerrors_$(GOOS)_$(GOARCH).go + ./mkunixsignals.sh $< > $@ || rm -f $@ + +signal_windows.go: ../syscall/ztypes_$(GOOS)_$(GOARCH).go + ./mkunixsignals.sh $< > $@ || rm -f $@ diff --git a/src/pkg/os/dir_plan9.go b/src/pkg/os/dir_plan9.go index d9514191d..bbc2cb647 100644 --- a/src/pkg/os/dir_plan9.go +++ b/src/pkg/os/dir_plan9.go @@ -9,35 +9,46 @@ import ( ) // Readdir reads the contents of the directory associated with file and -// returns an array of up to count FileInfo structures, in directory order. -// Subsequent calls on the same file will yield further FileInfos. -// A negative count means to read until EOF. -// Readdir returns the array and an Error, if any. -func (file *File) Readdir(count int) (fi []FileInfo, err Error) { +// returns an array of up to n FileInfo structures, as would be returned +// by Lstat, in directory order. Subsequent calls on the same file will yield +// further FileInfos. +// +// If n > 0, Readdir returns at most n FileInfo structures. In this case, if +// Readdirnames returns an empty slice, it will return a non-nil error +// explaining why. At the end of a directory, the error is os.EOF. +// +// If n <= 0, Readdir returns all the FileInfo from the directory in +// a single slice. In this case, if Readdir succeeds (reads all +// the way to the end of the directory), it returns the slice and a +// nil os.Error. If it encounters an error before the end of the +// directory, Readdir returns the FileInfo read until that point +// and a non-nil error. +func (file *File) Readdir(n int) (fi []FileInfo, err Error) { // If this file has no dirinfo, create one. if file.dirinfo == nil { file.dirinfo = new(dirInfo) } d := file.dirinfo - size := count - if size < 0 { + size := n + if size <= 0 { size = 100 + n = -1 } result := make([]FileInfo, 0, size) // Empty with room to grow. - for count != 0 { + for n != 0 { // Refill the buffer if necessary if d.bufp >= d.nbuf { d.bufp = 0 var e Error d.nbuf, e = file.Read(d.buf[:]) if e != nil && e != EOF { - return nil, &PathError{"readdir", file.name, e} + return result, &PathError{"readdir", file.name, e} } if e == EOF { break } if d.nbuf < syscall.STATFIXLEN { - return nil, &PathError{"readdir", file.name, Eshortstat} + return result, &PathError{"readdir", file.name, Eshortstat} } } @@ -45,39 +56,44 @@ func (file *File) Readdir(count int) (fi []FileInfo, err Error) { m, _ := gbit16(d.buf[d.bufp:]) m += 2 if m < syscall.STATFIXLEN { - return nil, &PathError{"readdir", file.name, Eshortstat} + return result, &PathError{"readdir", file.name, Eshortstat} } dir, e := UnmarshalDir(d.buf[d.bufp : d.bufp+int(m)]) if e != nil { - return nil, &PathError{"readdir", file.name, e} + return result, &PathError{"readdir", file.name, e} } var f FileInfo fileInfoFromStat(&f, dir) result = append(result, f) d.bufp += int(m) - count-- + n-- } - return result, nil -} -// Readdirnames returns an array of up to count file names residing in the -// directory associated with file. A negative count will return all of them. -// Readdir returns the array and an Error, if any. -func (file *File) Readdirnames(count int) (names []string, err Error) { - fi, e := file.Readdir(count) - - if e != nil { - return []string{}, e + if n >= 0 && len(result) == 0 { + return result, EOF } + return result, nil +} +// Readdirnames reads and returns a slice of names from the directory f. +// +// If n > 0, Readdirnames returns at most n names. In this case, if +// Readdirnames returns an empty slice, it will return a non-nil error +// explaining why. At the end of a directory, the error is os.EOF. +// +// If n <= 0, Readdirnames returns all the names from the directory in +// a single slice. In this case, if Readdirnames succeeds (reads all +// the way to the end of the directory), it returns the slice and a +// nil os.Error. If it encounters an error before the end of the +// directory, Readdirnames returns the names read until that point and +// a non-nil error. +func (file *File) Readdirnames(n int) (names []string, err Error) { + fi, err := file.Readdir(n) names = make([]string, len(fi)) - err = nil - for i := range fi { names[i] = fi[i].Name } - return } @@ -142,7 +158,7 @@ func pdir(b []byte, d *Dir) []byte { return b } -// UnmarshalDir reads a 9P Stat message from a 9P protocol message strored in b, +// UnmarshalDir reads a 9P Stat message from a 9P protocol message stored in b, // returning the corresponding Dir struct. func UnmarshalDir(b []byte) (d *Dir, err Error) { n := uint16(0) @@ -172,7 +188,7 @@ func UnmarshalDir(b []byte) (d *Dir, err Error) { return d, nil } -// gqid reads the qid part of a 9P Stat message from a 9P protocol message strored in b, +// gqid reads the qid part of a 9P Stat message from a 9P protocol message stored in b, // returning the corresponding Qid struct and the remaining slice of b. func gqid(b []byte) (Qid, []byte) { var q Qid @@ -190,25 +206,25 @@ func pqid(b []byte, q Qid) []byte { return b } -// gbit8 reads a byte-sized numeric value from a 9P protocol message strored in b, +// gbit8 reads a byte-sized numeric value from a 9P protocol message stored in b, // returning the value and the remaining slice of b. func gbit8(b []byte) (uint8, []byte) { return uint8(b[0]), b[1:] } -// gbit16 reads a 16-bit numeric value from a 9P protocol message strored in b, +// gbit16 reads a 16-bit numeric value from a 9P protocol message stored in b, // returning the value and the remaining slice of b. func gbit16(b []byte) (uint16, []byte) { return uint16(b[0]) | uint16(b[1])<<8, b[2:] } -// gbit32 reads a 32-bit numeric value from a 9P protocol message strored in b, +// gbit32 reads a 32-bit numeric value from a 9P protocol message stored in b, // returning the value and the remaining slice of b. func gbit32(b []byte) (uint32, []byte) { return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24, b[4:] } -// gbit64 reads a 64-bit numeric value from a 9P protocol message strored in b, +// gbit64 reads a 64-bit numeric value from a 9P protocol message stored in b, // returning the value and the remaining slice of b. func gbit64(b []byte) (uint64, []byte) { lo, b := gbit32(b) @@ -216,7 +232,7 @@ func gbit64(b []byte) (uint64, []byte) { return uint64(hi)<<32 | uint64(lo), b } -// gstring reads a string from a 9P protocol message strored in b, +// gstring reads a string from a 9P protocol message stored in b, // returning the value as a Go string and the remaining slice of b. func gstring(b []byte) (string, []byte) { n, b := gbit16(b) diff --git a/src/pkg/os/dir_unix.go b/src/pkg/os/dir_unix.go index f5b82230d..7835ed52b 100644 --- a/src/pkg/os/dir_unix.go +++ b/src/pkg/os/dir_unix.go @@ -12,30 +12,40 @@ const ( blockSize = 4096 ) -// Readdirnames reads the contents of the directory associated with file and -// returns an array of up to count names, in directory order. Subsequent -// calls on the same file will yield further names. -// A negative count means to read until EOF. -// Readdirnames returns the array and an Error, if any. -func (file *File) Readdirnames(count int) (names []string, err Error) { +// Readdirnames reads and returns a slice of names from the directory f. +// +// If n > 0, Readdirnames returns at most n names. In this case, if +// Readdirnames returns an empty slice, it will return a non-nil error +// explaining why. At the end of a directory, the error is os.EOF. +// +// If n <= 0, Readdirnames returns all the names from the directory in +// a single slice. In this case, if Readdirnames succeeds (reads all +// the way to the end of the directory), it returns the slice and a +// nil os.Error. If it encounters an error before the end of the +// directory, Readdirnames returns the names read until that point and +// a non-nil error. +func (f *File) Readdirnames(n int) (names []string, err Error) { // If this file has no dirinfo, create one. - if file.dirinfo == nil { - file.dirinfo = new(dirInfo) + if f.dirinfo == nil { + f.dirinfo = new(dirInfo) // The buffer must be at least a block long. - file.dirinfo.buf = make([]byte, blockSize) + f.dirinfo.buf = make([]byte, blockSize) } - d := file.dirinfo - size := count - if size < 0 { + d := f.dirinfo + + size := n + if size <= 0 { size = 100 + n = -1 } + names = make([]string, 0, size) // Empty with room to grow. - for count != 0 { + for n != 0 { // Refill the buffer if necessary if d.bufp >= d.nbuf { d.bufp = 0 var errno int - d.nbuf, errno = syscall.ReadDirent(file.fd, d.buf) + d.nbuf, errno = syscall.ReadDirent(f.fd, d.buf) if errno != 0 { return names, NewSyscallError("readdirent", errno) } @@ -46,9 +56,12 @@ func (file *File) Readdirnames(count int) (names []string, err Error) { // Drain the buffer var nb, nc int - nb, nc, names = syscall.ParseDirent(d.buf[d.bufp:d.nbuf], count, names) + nb, nc, names = syscall.ParseDirent(d.buf[d.bufp:d.nbuf], n, names) d.bufp += nb - count -= nc + n -= nc + } + if n >= 0 && len(names) == 0 { + return names, EOF } return names, nil } diff --git a/src/pkg/os/dir_windows.go b/src/pkg/os/dir_windows.go index 0d8267b59..d76e88fdb 100644 --- a/src/pkg/os/dir_windows.go +++ b/src/pkg/os/dir_windows.go @@ -4,14 +4,11 @@ package os -func (file *File) Readdirnames(count int) (names []string, err Error) { - fis, e := file.Readdir(count) - if e != nil { - return nil, e - } +func (file *File) Readdirnames(n int) (names []string, err Error) { + fis, err := file.Readdir(n) names = make([]string, len(fis)) for i, fi := range fis { names[i] = fi.Name } - return names, nil + return names, err } diff --git a/src/pkg/os/env.go b/src/pkg/os/env.go index 3a6d79dd0..3772c090b 100644 --- a/src/pkg/os/env.go +++ b/src/pkg/os/env.go @@ -6,6 +6,8 @@ package os +func setenv_c(k, v string) + // Expand replaces ${var} or $var in the string based on the mapping function. // Invocations of undefined variables are replaced with the empty string. func Expand(s string, mapping func(string) string) string { diff --git a/src/pkg/os/env_plan9.go b/src/pkg/os/env_plan9.go index 14df55ed0..1fed89f92 100644 --- a/src/pkg/os/env_plan9.go +++ b/src/pkg/os/env_plan9.go @@ -23,13 +23,18 @@ func Getenverror(key string) (value string, err Error) { } defer f.Close() - var buf [4096]byte - n, e := f.Read(buf[:len(buf)-1]) + l, _ := f.Seek(0, 2) + f.Seek(0, 0) + buf := make([]byte, l) + n, e := f.Read(buf) if iserror(e) { return "", ENOENV } - buf[n] = 0 - return string(buf[0:n]), nil + + if n > 0 && buf[n-1] == 0 { + buf = buf[:n-1] + } + return string(buf), nil } // Getenv retrieves the value of the environment variable named by the key. @@ -52,7 +57,7 @@ func Setenv(key, value string) Error { } defer f.Close() - _, e = f.Write(syscall.StringByteSlice(value)) + _, e = f.Write([]byte(value)) return nil } diff --git a/src/pkg/os/env_unix.go b/src/pkg/os/env_unix.go index e7e1c3b90..8aa71e83a 100644 --- a/src/pkg/os/env_unix.go +++ b/src/pkg/os/env_unix.go @@ -29,6 +29,8 @@ func copyenv() { } } +var envLock sync.RWMutex + // Getenverror retrieves the value of the environment variable named by the key. // It returns the value and an error, if any. func Getenverror(key string) (value string, err Error) { @@ -37,6 +39,10 @@ func Getenverror(key string) (value string, err Error) { if len(key) == 0 { return "", EINVAL } + + envLock.RLock() + defer envLock.RUnlock() + v, ok := env[key] if !ok { return "", ENOENV @@ -55,35 +61,43 @@ func Getenv(key string) string { // It returns an Error, if any. func Setenv(key, value string) Error { once.Do(copyenv) - if len(key) == 0 { return EINVAL } + + envLock.Lock() + defer envLock.Unlock() + env[key] = value + setenv_c(key, value) // is a no-op if cgo isn't loaded return nil } // Clearenv deletes all environment variables. func Clearenv() { once.Do(copyenv) // prevent copyenv in Getenv/Setenv + + envLock.Lock() + defer envLock.Unlock() + env = make(map[string]string) + + // TODO(bradfitz): pass through to C } // Environ returns an array of strings representing the environment, // in the form "key=value". func Environ() []string { once.Do(copyenv) + envLock.RLock() + defer envLock.RUnlock() a := make([]string, len(env)) i := 0 for k, v := range env { - // check i < len(a) for safety, - // in case env is changing underfoot. - if i < len(a) { - a[i] = k + "=" + v - i++ - } + a[i] = k + "=" + v + i++ } - return a[0:i] + return a } // TempDir returns the default directory to use for temporary files. diff --git a/src/pkg/os/error_posix.go b/src/pkg/os/error_posix.go index 0ee34e4b0..d43f1786d 100644 --- a/src/pkg/os/error_posix.go +++ b/src/pkg/os/error_posix.go @@ -13,7 +13,7 @@ type Errno int64 func (e Errno) String() string { return syscall.Errstr(int(e)) } func (e Errno) Temporary() bool { - return e == Errno(syscall.EINTR) || e.Timeout() + return e == Errno(syscall.EINTR) || e == Errno(syscall.EMFILE) || e.Timeout() } func (e Errno) Timeout() bool { diff --git a/src/pkg/os/exec_posix.go b/src/pkg/os/exec_posix.go index 9102dc0a4..bf992ef42 100644 --- a/src/pkg/os/exec_posix.go +++ b/src/pkg/os/exec_posix.go @@ -4,7 +4,25 @@ package os -import "syscall" +import ( + "runtime" + "syscall" +) + +// A Signal can represent any operating system signal. +type Signal interface { + String() string +} + +type UnixSignal int32 + +func (sig UnixSignal) String() string { + s := runtime.Signame(int32(sig)) + if len(s) > 0 { + return s + } + return "UnixSignal" +} // StartProcess starts a new process with the program, arguments and attributes // specified by name, argv and attr. @@ -34,6 +52,11 @@ func StartProcess(name string, argv []string, attr *ProcAttr) (p *Process, err E return newProcess(pid, h), nil } +// Kill causes the Process to exit immediately. +func (p *Process) Kill() Error { + return p.Signal(SIGKILL) +} + // Exec replaces the current process with an execution of the // named binary, with arguments argv and environment envv. // If successful, Exec never returns. If it fails, it returns an Error. diff --git a/src/pkg/os/exec_unix.go b/src/pkg/os/exec_unix.go index 8990d6a97..cf5ea9b61 100644 --- a/src/pkg/os/exec_unix.go +++ b/src/pkg/os/exec_unix.go @@ -45,6 +45,14 @@ func (p *Process) Wait(options int) (w *Waitmsg, err Error) { return w, nil } +// Signal sends a signal to the Process. +func (p *Process) Signal(sig Signal) Error { + if e := syscall.Kill(p.Pid, int(sig.(UnixSignal))); e != 0 { + return Errno(e) + } + return nil +} + // Release releases any resources associated with the Process. func (p *Process) Release() Error { // NOOP for unix. diff --git a/src/pkg/os/exec_windows.go b/src/pkg/os/exec_windows.go index ae8ffeab2..bac33b908 100644 --- a/src/pkg/os/exec_windows.go +++ b/src/pkg/os/exec_windows.go @@ -20,13 +20,23 @@ func (p *Process) Wait(options int) (w *Waitmsg, err Error) { return nil, ErrorString("os: unexpected result from WaitForSingleObject") } var ec uint32 - e = syscall.GetExitCodeProcess(uint32(p.handle), &ec) + e = syscall.GetExitCodeProcess(int32(p.handle), &ec) if e != 0 { return nil, NewSyscallError("GetExitCodeProcess", e) } return &Waitmsg{p.Pid, syscall.WaitStatus{s, ec}, new(syscall.Rusage)}, nil } +// Signal sends a signal to the Process. +func (p *Process) Signal(sig Signal) Error { + switch sig.(UnixSignal) { + case SIGKILL: + e := syscall.TerminateProcess(int32(p.handle), 1) + return NewSyscallError("TerminateProcess", e) + } + return Errno(syscall.EWINDOWS) +} + func (p *Process) Release() Error { if p.handle == -1 { return EINVAL diff --git a/src/pkg/os/file_unix.go b/src/pkg/os/file_unix.go index 2fb28df65..def9b3bf0 100644 --- a/src/pkg/os/file_unix.go +++ b/src/pkg/os/file_unix.go @@ -69,21 +69,27 @@ func (file *File) Stat() (fi *FileInfo, err Error) { } // Readdir reads the contents of the directory associated with file and -// returns an array of up to count FileInfo structures, as would be returned -// by Lstat, in directory order. Subsequent calls on the same file will yield +// returns an array of up to n FileInfo structures, as would be returned +// by Lstat, in directory order. Subsequent calls on the same file will yield // further FileInfos. -// A negative count means to read until EOF. -// Readdir returns the array and an Error, if any. -func (file *File) Readdir(count int) (fi []FileInfo, err Error) { +// +// If n > 0, Readdir returns at most n FileInfo structures. In this case, if +// Readdir returns an empty slice, it will return a non-nil error +// explaining why. At the end of a directory, the error is os.EOF. +// +// If n <= 0, Readdir returns all the FileInfo from the directory in +// a single slice. In this case, if Readdir succeeds (reads all +// the way to the end of the directory), it returns the slice and a +// nil os.Error. If it encounters an error before the end of the +// directory, Readdir returns the FileInfo read until that point +// and a non-nil error. +func (file *File) Readdir(n int) (fi []FileInfo, err Error) { dirname := file.name if dirname == "" { dirname = "." } dirname += "/" - names, err1 := file.Readdirnames(count) - if err1 != nil { - return nil, err1 - } + names, err := file.Readdirnames(n) fi = make([]FileInfo, len(names)) for i, filename := range names { fip, err := Lstat(dirname + filename) diff --git a/src/pkg/os/file_windows.go b/src/pkg/os/file_windows.go index 95f60b735..80886f6f5 100644 --- a/src/pkg/os/file_windows.go +++ b/src/pkg/os/file_windows.go @@ -123,12 +123,21 @@ func (file *File) Stat() (fi *FileInfo, err Error) { } // Readdir reads the contents of the directory associated with file and -// returns an array of up to count FileInfo structures, as would be returned -// by Lstat, in directory order. Subsequent calls on the same file will yield +// returns an array of up to n FileInfo structures, as would be returned +// by Lstat, in directory order. Subsequent calls on the same file will yield // further FileInfos. -// A negative count means to read until EOF. -// Readdir returns the array and an Error, if any. -func (file *File) Readdir(count int) (fi []FileInfo, err Error) { +// +// If n > 0, Readdir returns at most n FileInfo structures. In this case, if +// Readdir returns an empty slice, it will return a non-nil error +// explaining why. At the end of a directory, the error is os.EOF. +// +// If n <= 0, Readdir returns all the FileInfo from the directory in +// a single slice. In this case, if Readdir succeeds (reads all +// the way to the end of the directory), it returns the slice and a +// nil os.Error. If it encounters an error before the end of the +// directory, Readdir returns the FileInfo read until that point +// and a non-nil error. +func (file *File) Readdir(n int) (fi []FileInfo, err Error) { if file == nil || file.fd < 0 { return nil, EINVAL } @@ -136,12 +145,14 @@ func (file *File) Readdir(count int) (fi []FileInfo, err Error) { return nil, &PathError{"Readdir", file.name, ENOTDIR} } di := file.dirinfo - size := count - if size < 0 { + wantAll := n <= 0 + size := n + if wantAll { + n = -1 size = 100 } fi = make([]FileInfo, 0, size) // Empty with room to grow. - for count != 0 { + for n != 0 { if di.usefirststat { di.usefirststat = false } else { @@ -150,7 +161,11 @@ func (file *File) Readdir(count int) (fi []FileInfo, err Error) { if e == syscall.ERROR_NO_MORE_FILES { break } else { - return nil, &PathError{"FindNextFile", file.name, Errno(e)} + err = &PathError{"FindNextFile", file.name, Errno(e)} + if !wantAll { + fi = nil + } + return } } } @@ -159,9 +174,12 @@ func (file *File) Readdir(count int) (fi []FileInfo, err Error) { if f.Name == "." || f.Name == ".." { // Useless names continue } - count-- + n-- fi = append(fi, f) } + if !wantAll && len(fi) == 0 { + return fi, EOF + } return fi, nil } diff --git a/src/pkg/os/signal/mkunix.sh b/src/pkg/os/mkunixsignals.sh index 653b01664..6ec764cbd 100755 --- a/src/pkg/os/signal/mkunix.sh +++ b/src/pkg/os/mkunixsignals.sh @@ -8,7 +8,7 @@ echo '// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT' echo cat <<EOH -package signal +package os import ( "syscall" diff --git a/src/pkg/os/os_test.go b/src/pkg/os/os_test.go index 65475c118..8eabdee6b 100644 --- a/src/pkg/os/os_test.go +++ b/src/pkg/os/os_test.go @@ -236,11 +236,14 @@ func smallReaddirnames(file *File, length int, t *testing.T) []string { count := 0 for { d, err := file.Readdirnames(1) + if err == EOF { + break + } if err != nil { - t.Fatalf("readdir %q failed: %v", file.Name(), err) + t.Fatalf("readdirnames %q failed: %v", file.Name(), err) } if len(d) == 0 { - break + t.Fatalf("readdirnames %q returned empty slice and no error", file.Name()) } names[count] = d[0] count++ @@ -283,6 +286,78 @@ func TestReaddirnamesOneAtATime(t *testing.T) { } } +func TestReaddirNValues(t *testing.T) { + if testing.Short() { + t.Logf("test.short; skipping") + return + } + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("TempDir: %v", err) + } + defer RemoveAll(dir) + for i := 1; i <= 105; i++ { + f, err := Create(filepath.Join(dir, fmt.Sprintf("%d", i))) + if err != nil { + t.Fatalf("Create: %v", err) + } + f.Write([]byte(strings.Repeat("X", i))) + f.Close() + } + + var d *File + openDir := func() { + var err Error + d, err = Open(dir) + if err != nil { + t.Fatalf("Open directory: %v", err) + } + } + + readDirExpect := func(n, want int, wantErr Error) { + fi, err := d.Readdir(n) + if err != wantErr { + t.Fatalf("Readdir of %d got error %v, want %v", n, err, wantErr) + } + if g, e := len(fi), want; g != e { + t.Errorf("Readdir of %d got %d files, want %d", n, g, e) + } + } + + readDirNamesExpect := func(n, want int, wantErr Error) { + fi, err := d.Readdirnames(n) + if err != wantErr { + t.Fatalf("Readdirnames of %d got error %v, want %v", n, err, wantErr) + } + if g, e := len(fi), want; g != e { + t.Errorf("Readdirnames of %d got %d files, want %d", n, g, e) + } + } + + for _, fn := range []func(int, int, Error){readDirExpect, readDirNamesExpect} { + // Test the slurp case + openDir() + fn(0, 105, nil) + fn(0, 0, nil) + d.Close() + + // Slurp with -1 instead + openDir() + fn(-1, 105, nil) + fn(-2, 0, nil) + fn(0, 0, nil) + d.Close() + + // Test the bounded case + openDir() + fn(1, 1, nil) + fn(2, 2, nil) + fn(105, 102, nil) // and tests buffer >100 case + fn(3, 0, EOF) + d.Close() + } +} + func TestHardLink(t *testing.T) { // Hardlinks are not supported under windows. if syscall.OS == "windows" { @@ -439,7 +514,8 @@ func exec(t *testing.T, dir, cmd string, args []string, expect string) { var b bytes.Buffer io.Copy(&b, r) output := b.String() - if output != expect { + // Accept /usr prefix because Solaris /bin is symlinked to /usr/bin. + if output != expect && output != "/usr"+expect { t.Errorf("exec %q returned %q wanted %q", strings.Join(append([]string{cmd}, args...), " "), output, expect) } @@ -914,7 +990,15 @@ func TestAppend(t *testing.T) { } s = writeFile(t, f, O_CREATE|O_APPEND|O_RDWR, "new&append") if s != "new&append" { - t.Fatalf("writeFile: have %q want %q", s, "new&append") + t.Fatalf("writeFile: after append have %q want %q", s, "new&append") + } + s = writeFile(t, f, O_CREATE|O_RDWR, "old") + if s != "old&append" { + t.Fatalf("writeFile: after create have %q want %q", s, "old&append") + } + s = writeFile(t, f, O_CREATE|O_TRUNC|O_RDWR, "new") + if s != "new" { + t.Fatalf("writeFile: after truncate have %q want %q", s, "new") } } diff --git a/src/pkg/os/path.go b/src/pkg/os/path.go index 0eb3ee503..7b93036aa 100644 --- a/src/pkg/os/path.go +++ b/src/pkg/os/path.go @@ -24,12 +24,12 @@ func MkdirAll(path string, perm uint32) Error { // Doesn't already exist; make sure parent does. i := len(path) - for i > 0 && path[i-1] == '/' { // Skip trailing slashes. + for i > 0 && IsPathSeparator(path[i-1]) { // Skip trailing path separator. i-- } j := i - for j > 0 && path[j-1] != '/' { // Scan backward over element. + for j > 0 && !IsPathSeparator(path[j-1]) { // Scan backward over element. j-- } @@ -90,11 +90,14 @@ func RemoveAll(path string) Error { for { names, err1 := fd.Readdirnames(100) for _, name := range names { - err1 := RemoveAll(path + "/" + name) + err1 := RemoveAll(path + string(PathSeparator) + name) if err == nil { err = err1 } } + if err1 == EOF { + break + } // If Readdirnames returned an error, use it. if err == nil { err = err1 diff --git a/src/pkg/os/path_plan9.go b/src/pkg/os/path_plan9.go new file mode 100644 index 000000000..3121b7bc7 --- /dev/null +++ b/src/pkg/os/path_plan9.go @@ -0,0 +1,15 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +const ( + PathSeparator = '/' // OS-specific path separator + PathListSeparator = 0 // OS-specific path list separator +) + +// IsPathSeparator returns true if c is a directory separator character. +func IsPathSeparator(c uint8) bool { + return PathSeparator == c +} diff --git a/src/pkg/os/path_test.go b/src/pkg/os/path_test.go index 483bb6395..d58945aab 100644 --- a/src/pkg/os/path_test.go +++ b/src/pkg/os/path_test.go @@ -6,6 +6,7 @@ package os_test import ( . "os" + "path/filepath" "testing" "runtime" "syscall" @@ -29,10 +30,11 @@ func TestMkdirAll(t *testing.T) { // Make file. fpath := path + "/file" - _, err = Create(fpath) + f, err := Create(fpath) if err != nil { t.Fatalf("create %q: %s", fpath, err) } + defer f.Close() // Can't make directory named after file. err = MkdirAll(fpath, 0777) @@ -43,8 +45,8 @@ func TestMkdirAll(t *testing.T) { if !ok { t.Fatalf("MkdirAll %q returned %T, not *PathError", fpath, err) } - if perr.Path != fpath { - t.Fatalf("MkdirAll %q returned wrong error path: %q not %q", fpath, perr.Path, fpath) + if filepath.Clean(perr.Path) != filepath.Clean(fpath) { + t.Fatalf("MkdirAll %q returned wrong error path: %q not %q", fpath, filepath.Clean(perr.Path), filepath.Clean(fpath)) } // Can't make subdirectory of file. @@ -57,8 +59,16 @@ func TestMkdirAll(t *testing.T) { if !ok { t.Fatalf("MkdirAll %q returned %T, not *PathError", ffpath, err) } - if perr.Path != fpath { - t.Fatalf("MkdirAll %q returned wrong error path: %q not %q", ffpath, perr.Path, fpath) + if filepath.Clean(perr.Path) != filepath.Clean(fpath) { + t.Fatalf("MkdirAll %q returned wrong error path: %q not %q", ffpath, filepath.Clean(perr.Path), filepath.Clean(fpath)) + } + + if syscall.OS == "windows" { + path := `_test\_TestMkdirAll_\dir\.\dir2\` + err := MkdirAll(path, 0777) + if err != nil { + t.Fatalf("MkdirAll %q: %s", path, err) + } } } diff --git a/src/pkg/os/path_unix.go b/src/pkg/os/path_unix.go new file mode 100644 index 000000000..0d327cddd --- /dev/null +++ b/src/pkg/os/path_unix.go @@ -0,0 +1,15 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +const ( + PathSeparator = '/' // OS-specific path separator + PathListSeparator = ':' // OS-specific path list separator +) + +// IsPathSeparator returns true if c is a directory separator character. +func IsPathSeparator(c uint8) bool { + return PathSeparator == c +} diff --git a/src/pkg/os/path_windows.go b/src/pkg/os/path_windows.go new file mode 100644 index 000000000..8740a9e61 --- /dev/null +++ b/src/pkg/os/path_windows.go @@ -0,0 +1,16 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +const ( + PathSeparator = '\\' // OS-specific path separator + PathListSeparator = ':' // OS-specific path list separator +) + +// IsPathSeparator returns true if c is a directory separator character. +func IsPathSeparator(c uint8) bool { + // NOTE: Windows accept / as path separator. + return c == '\\' || c == '/' +} diff --git a/src/pkg/os/signal/Makefile b/src/pkg/os/signal/Makefile index 013b91a85..26f58760e 100644 --- a/src/pkg/os/signal/Makefile +++ b/src/pkg/os/signal/Makefile @@ -7,11 +7,5 @@ include ../../../Make.inc TARG=os/signal GOFILES=\ signal.go\ - unix.go\ - -CLEANFILES+=unix.go include ../../../Make.pkg - -unix.go: ../../syscall/zerrors_$(GOOS)_$(GOARCH).go - ./mkunix.sh $< > $@ || rm -f $@ diff --git a/src/pkg/os/signal/signal.go b/src/pkg/os/signal/signal.go index 666c03e73..520f3f8a9 100644 --- a/src/pkg/os/signal/signal.go +++ b/src/pkg/os/signal/signal.go @@ -6,35 +6,20 @@ package signal import ( + "os" "runtime" - "strconv" ) -// A Signal can represent any operating system signal. -type Signal interface { - String() string -} - -type UnixSignal int32 - -func (sig UnixSignal) String() string { - s := runtime.Signame(int32(sig)) - if len(s) > 0 { - return s - } - return "Signal " + strconv.Itoa(int(sig)) -} - // Incoming is the global signal channel. // All signals received by the program will be delivered to this channel. -var Incoming <-chan Signal +var Incoming <-chan os.Signal -func process(ch chan<- Signal) { +func process(ch chan<- os.Signal) { for { var mask uint32 = runtime.Sigrecv() for sig := uint(0); sig < 32; sig++ { if mask&(1<<sig) != 0 { - ch <- UnixSignal(sig) + ch <- os.UnixSignal(sig) } } } @@ -42,7 +27,7 @@ func process(ch chan<- Signal) { func init() { runtime.Siginit() - ch := make(chan Signal) // Done here so Incoming can have type <-chan Signal + ch := make(chan os.Signal) // Done here so Incoming can have type <-chan Signal Incoming = ch go process(ch) } diff --git a/src/pkg/os/signal/signal_test.go b/src/pkg/os/signal/signal_test.go index f2679f14d..00eb29578 100644 --- a/src/pkg/os/signal/signal_test.go +++ b/src/pkg/os/signal/signal_test.go @@ -5,6 +5,7 @@ package signal import ( + "os" "syscall" "testing" ) @@ -13,7 +14,7 @@ func TestSignal(t *testing.T) { // Send this process a SIGHUP. syscall.Syscall(syscall.SYS_KILL, uintptr(syscall.Getpid()), syscall.SIGHUP, 0) - if sig := (<-Incoming).(UnixSignal); sig != SIGHUP { - t.Errorf("signal was %v, want %v", sig, SIGHUP) + if sig := (<-Incoming).(os.UnixSignal); sig != os.SIGHUP { + t.Errorf("signal was %v, want %v", sig, os.SIGHUP) } } diff --git a/src/pkg/os/user/user_test.go b/src/pkg/os/user/user_test.go index 2c142bf18..ee917b57a 100644 --- a/src/pkg/os/user/user_test.go +++ b/src/pkg/os/user/user_test.go @@ -42,7 +42,7 @@ func TestLookup(t *testing.T) { } fi, err := os.Stat(u.HomeDir) if err != nil || !fi.IsDirectory() { - t.Errorf("expected a valid HomeDir; stat(%q): err=%v, IsDirectory=%v", err, fi.IsDirectory()) + t.Errorf("expected a valid HomeDir; stat(%q): err=%v, IsDirectory=%v", u.HomeDir, err, fi.IsDirectory()) } if u.Username == "" { t.Fatalf("didn't get a username") @@ -56,6 +56,6 @@ func TestLookup(t *testing.T) { if !reflect.DeepEqual(u, un) { t.Errorf("Lookup by userid vs. name didn't match\n"+ "LookupId(%d): %#v\n"+ - "Lookup(%q): %#v\n",uid, u, u.Username, un) + "Lookup(%q): %#v\n", uid, u, u.Username, un) } } diff --git a/src/pkg/path/filepath/match.go b/src/pkg/path/filepath/match.go index a05bb5f7e..9c344309d 100644 --- a/src/pkg/path/filepath/match.go +++ b/src/pkg/path/filepath/match.go @@ -124,9 +124,8 @@ func matchChunk(chunk, s string) (rest string, ok bool, err os.Error) { s = s[n:] chunk = chunk[1:] // possibly negated - notNegated := true - if len(chunk) > 0 && chunk[0] == '^' { - notNegated = false + negated := chunk[0] == '^' + if negated { chunk = chunk[1:] } // parse all ranges @@ -152,7 +151,7 @@ func matchChunk(chunk, s string) (rest string, ok bool, err os.Error) { } nrange++ } - if match != notNegated { + if match == negated { return } diff --git a/src/pkg/path/filepath/match_test.go b/src/pkg/path/filepath/match_test.go index 43e1c1cc2..a1c8333f3 100644 --- a/src/pkg/path/filepath/match_test.go +++ b/src/pkg/path/filepath/match_test.go @@ -69,6 +69,13 @@ var matchTests = []MatchTest{ {"*x", "xxx", true, nil}, } +func errp(e os.Error) string { + if e == nil { + return "<nil>" + } + return e.String() +} + func TestMatch(t *testing.T) { if runtime.GOOS == "windows" { // XXX: Don't pass for windows. @@ -77,7 +84,7 @@ func TestMatch(t *testing.T) { for _, tt := range matchTests { ok, err := Match(tt.pattern, tt.s) if ok != tt.match || err != tt.err { - t.Errorf("Match(%#q, %#q) = %v, %v want %v, nil", tt.pattern, tt.s, ok, err, tt.match) + t.Errorf("Match(%#q, %#q) = %v, %q want %v, %q", tt.pattern, tt.s, ok, errp(err), tt.match, errp(tt.err)) } } } diff --git a/src/pkg/path/filepath/path.go b/src/pkg/path/filepath/path.go index 541a23306..dcd8017ad 100644 --- a/src/pkg/path/filepath/path.go +++ b/src/pkg/path/filepath/path.go @@ -9,13 +9,14 @@ package filepath import ( "bytes" "os" + "runtime" "sort" "strings" ) const ( - SeparatorString = string(Separator) - ListSeparatorString = string(ListSeparator) + Separator = os.PathSeparator + ListSeparator = os.PathListSeparator ) // Clean returns the shortest path name equivalent to path @@ -60,20 +61,20 @@ func Clean(path string) string { for r < n { switch { - case isSeparator(path[r]): + case os.IsPathSeparator(path[r]): // empty path element r++ - case path[r] == '.' && (r+1 == n || isSeparator(path[r+1])): + case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])): // . element r++ - case path[r] == '.' && path[r+1] == '.' && (r+2 == n || isSeparator(path[r+2])): + case path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])): // .. element: remove to last separator r += 2 switch { case w > dotdot: // can backtrack w-- - for w > dotdot && !isSeparator(buf[w]) { + for w > dotdot && !os.IsPathSeparator(buf[w]) { w-- } case !rooted: @@ -96,7 +97,7 @@ func Clean(path string) string { w++ } // copy element - for ; r < n && !isSeparator(path[r]); r++ { + for ; r < n && !os.IsPathSeparator(path[r]); r++ { buf[w] = path[r] w++ } @@ -118,7 +119,7 @@ func ToSlash(path string) string { if Separator == '/' { return path } - return strings.Replace(path, SeparatorString, "/", -1) + return strings.Replace(path, string(Separator), "/", -1) } // FromSlash returns the result of replacing each slash ('/') character @@ -127,7 +128,7 @@ func FromSlash(path string) string { if Separator == '/' { return path } - return strings.Replace(path, "/", SeparatorString, -1) + return strings.Replace(path, "/", string(Separator), -1) } // SplitList splits a list of paths joined by the OS-specific ListSeparator. @@ -135,7 +136,7 @@ func SplitList(path string) []string { if path == "" { return []string{} } - return strings.Split(path, ListSeparatorString, -1) + return strings.Split(path, string(ListSeparator), -1) } // Split splits path immediately following the final Separator, @@ -144,7 +145,7 @@ func SplitList(path string) []string { // and file set to path. func Split(path string) (dir, file string) { i := len(path) - 1 - for i >= 0 && !isSeparator(path[i]) { + for i >= 0 && !os.IsPathSeparator(path[i]) { i-- } return path[:i+1], path[i+1:] @@ -155,7 +156,7 @@ func Split(path string) (dir, file string) { func Join(elem ...string) string { for i, e := range elem { if e != "" { - return Clean(strings.Join(elem[i:], SeparatorString)) + return Clean(strings.Join(elem[i:], string(Separator))) } } return "" @@ -166,7 +167,7 @@ func Join(elem ...string) string { // in the final element of path; it is empty if there is // no dot. func Ext(path string) string { - for i := len(path) - 1; i >= 0 && !isSeparator(path[i]); i-- { + for i := len(path) - 1; i >= 0 && !os.IsPathSeparator(path[i]); i-- { if path[i] == '.' { return path[i:] } @@ -178,6 +179,14 @@ func Ext(path string) string { // links. // If path is relative it will be evaluated relative to the current directory. func EvalSymlinks(path string) (string, os.Error) { + if runtime.GOOS == "windows" { + // Symlinks are not supported under windows. + _, err := os.Lstat(path) + if err != nil { + return "", err + } + return Clean(path), nil + } const maxIter = 255 originalPath := path // consume path by taking each frontmost path element, @@ -225,7 +234,7 @@ func EvalSymlinks(path string) (string, os.Error) { if IsAbs(dest) { b.Reset() } - path = dest + SeparatorString + path + path = dest + string(Separator) + path } return Clean(b.String()), nil } @@ -236,7 +245,7 @@ func EvalSymlinks(path string) (string, os.Error) { // path name for a given file is not guaranteed to be unique. func Abs(path string) (string, os.Error) { if IsAbs(path) { - return path, nil + return Clean(path), nil } wd, err := os.Getwd() if err != nil { @@ -330,12 +339,12 @@ func Base(path string) string { return "." } // Strip trailing slashes. - for len(path) > 0 && isSeparator(path[len(path)-1]) { + for len(path) > 0 && os.IsPathSeparator(path[len(path)-1]) { path = path[0 : len(path)-1] } // Find the last element i := len(path) - 1 - for i >= 0 && !isSeparator(path[i]) { + for i >= 0 && !os.IsPathSeparator(path[i]) { i-- } if i >= 0 { @@ -343,7 +352,7 @@ func Base(path string) string { } // If empty now, it had only slashes. if path == "" { - return SeparatorString + return string(Separator) } return path } diff --git a/src/pkg/path/filepath/path_plan9.go b/src/pkg/path/filepath/path_plan9.go index e40008364..47990e0fe 100644 --- a/src/pkg/path/filepath/path_plan9.go +++ b/src/pkg/path/filepath/path_plan9.go @@ -6,16 +6,6 @@ package filepath import "strings" -const ( - Separator = '/' // OS-specific path separator - ListSeparator = 0 // OS-specific path list separator -) - -// isSeparator returns true if c is a directory separator character. -func isSeparator(c uint8) bool { - return Separator == c -} - // IsAbs returns true if the path is absolute. func IsAbs(path string) bool { return strings.HasPrefix(path, "/") || strings.HasPrefix(path, "#") diff --git a/src/pkg/path/filepath/path_test.go b/src/pkg/path/filepath/path_test.go index b3b6eb5ab..6a5dd5b00 100644 --- a/src/pkg/path/filepath/path_test.go +++ b/src/pkg/path/filepath/path_test.go @@ -315,7 +315,10 @@ func TestWalk(t *testing.T) { } checkMarks(t) - if os.Getuid() > 0 { + // Test permission errors. Only possible if we're not root + // and only on some file systems (AFS, FAT). To avoid errors during + // all.bash on those file systems, skip during gotest -short. + if os.Getuid() > 0 && !testing.Short() { // introduce 2 errors: chmod top-level directories to 0 os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0) os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0) @@ -440,48 +443,64 @@ var EvalSymlinksTests = []EvalSymlinksTest{ {"test/link2/link3/test", "test"}, } -func TestEvalSymlinks(t *testing.T) { - // Symlinks are not supported under windows. - if runtime.GOOS == "windows" { - return +var EvalSymlinksAbsWindowsTests = []EvalSymlinksTest{ + {`c:\`, `c:\`}, +} + +func testEvalSymlinks(t *testing.T, tests []EvalSymlinksTest) { + for _, d := range tests { + if p, err := filepath.EvalSymlinks(d.path); err != nil { + t.Errorf("EvalSymlinks(%q) error: %v", d.path, err) + } else if filepath.Clean(p) != filepath.Clean(d.dest) { + t.Errorf("EvalSymlinks(%q)=%q, want %q", d.path, p, d.dest) + } } +} + +func TestEvalSymlinks(t *testing.T) { defer os.RemoveAll("test") for _, d := range EvalSymlinksTestDirs { var err os.Error if d.dest == "" { err = os.Mkdir(d.path, 0755) } else { - err = os.Symlink(d.dest, d.path) + if runtime.GOOS != "windows" { + err = os.Symlink(d.dest, d.path) + } } if err != nil { t.Fatal(err) } } - // relative - for _, d := range EvalSymlinksTests { - if p, err := filepath.EvalSymlinks(d.path); err != nil { - t.Errorf("EvalSymlinks(%q) error: %v", d.path, err) - } else if p != d.dest { - t.Errorf("EvalSymlinks(%q)=%q, want %q", d.path, p, d.dest) + var tests []EvalSymlinksTest + if runtime.GOOS == "windows" { + for _, d := range EvalSymlinksTests { + if d.path == d.dest { + // will test only real files and directories + tests = append(tests, d) + } } + } else { + tests = EvalSymlinksTests } + // relative + testEvalSymlinks(t, tests) // absolute goroot, err := filepath.EvalSymlinks(os.Getenv("GOROOT")) if err != nil { t.Fatalf("EvalSymlinks(%q) error: %v", os.Getenv("GOROOT"), err) } testroot := filepath.Join(goroot, "src", "pkg", "path", "filepath") - for _, d := range EvalSymlinksTests { - a := EvalSymlinksTest{ - filepath.Join(testroot, d.path), - filepath.Join(testroot, d.dest), - } - if p, err := filepath.EvalSymlinks(a.path); err != nil { - t.Errorf("EvalSymlinks(%q) error: %v", a.path, err) - } else if p != a.dest { - t.Errorf("EvalSymlinks(%q)=%q, want %q", a.path, p, a.dest) + for i, d := range tests { + tests[i].path = filepath.Join(testroot, d.path) + tests[i].dest = filepath.Join(testroot, d.dest) + } + if runtime.GOOS == "windows" { + for _, d := range EvalSymlinksAbsWindowsTests { + tests = append(tests, d) } } + testEvalSymlinks(t, tests) } // Test paths relative to $GOROOT/src @@ -493,6 +512,7 @@ var abstests = []string{ // Already absolute "$GOROOT/src/Make.pkg", + "$GOROOT/src/../src/Make.pkg", } func TestAbs(t *testing.T) { @@ -521,5 +541,8 @@ func TestAbs(t *testing.T) { if !filepath.IsAbs(abspath) { t.Errorf("Abs(%q)=%q, not an absolute path", path, abspath) } + if filepath.IsAbs(path) && abspath != filepath.Clean(path) { + t.Errorf("Abs(%q)=%q, isn't clean", path, abspath) + } } } diff --git a/src/pkg/path/filepath/path_unix.go b/src/pkg/path/filepath/path_unix.go index f8ac248fb..ea555fc0e 100644 --- a/src/pkg/path/filepath/path_unix.go +++ b/src/pkg/path/filepath/path_unix.go @@ -6,16 +6,6 @@ package filepath import "strings" -const ( - Separator = '/' // OS-specific path separator - ListSeparator = ':' // OS-specific path list separator -) - -// isSeparator returns true if c is a directory separator character. -func isSeparator(c uint8) bool { - return Separator == c -} - // IsAbs returns true if the path is absolute. func IsAbs(path string) bool { return strings.HasPrefix(path, "/") diff --git a/src/pkg/path/filepath/path_windows.go b/src/pkg/path/filepath/path_windows.go index dbd1c1e40..35302eb1a 100644 --- a/src/pkg/path/filepath/path_windows.go +++ b/src/pkg/path/filepath/path_windows.go @@ -4,20 +4,11 @@ package filepath -const ( - Separator = '\\' // OS-specific path separator - ListSeparator = ':' // OS-specific path list separator -) - -// isSeparator returns true if c is a directory separator character. -func isSeparator(c uint8) bool { - // NOTE: Windows accept / as path separator. - return c == '\\' || c == '/' -} +import "os" // IsAbs returns true if the path is absolute. func IsAbs(path string) bool { - return path != "" && (volumeName(path) != "" || isSeparator(path[0])) + return path != "" && (volumeName(path) != "" || os.IsPathSeparator(path[0])) } // volumeName return leading volume name. @@ -28,7 +19,7 @@ func volumeName(path string) string { } // with drive letter c := path[0] - if len(path) > 2 && path[1] == ':' && isSeparator(path[2]) && + if len(path) > 2 && path[1] == ':' && os.IsPathSeparator(path[2]) && ('0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') { return path[0:2] diff --git a/src/pkg/rand/rand_test.go b/src/pkg/rand/rand_test.go index 2476ebaf6..a689da848 100644 --- a/src/pkg/rand/rand_test.go +++ b/src/pkg/rand/rand_test.go @@ -197,7 +197,7 @@ func initNorm() (testKn []uint32, testWn, testFn []float32) { const m1 = 1 << 31 var ( dn float64 = rn - tn = dn + tn = dn vn float64 = 9.91256303526217e-3 ) @@ -226,7 +226,7 @@ func initExp() (testKe []uint32, testWe, testFe []float32) { const m2 = 1 << 32 var ( de float64 = re - te = de + te = de ve float64 = 3.9496598225815571993e-3 ) diff --git a/src/pkg/reflect/all_test.go b/src/pkg/reflect/all_test.go index e20e03f50..c83a9b75f 100644 --- a/src/pkg/reflect/all_test.go +++ b/src/pkg/reflect/all_test.go @@ -182,9 +182,7 @@ var valueTests = []pair{ }), "struct { c chan *int32; d float32 }{chan *int32, 0}", }, - {new(struct { - c func(chan *integer, *int8) - }), + {new(struct{ c func(chan *integer, *int8) }), "struct { c func(chan *reflect_test.integer, *int8) }{func(chan *reflect_test.integer, *int8)(0)}", }, {new(struct { @@ -581,13 +579,7 @@ func TestCopyArray(t *testing.T) { } for i := len(a); i < len(b); i++ { if b[i] != c[i] { - if i < len(a) { - t.Errorf("(ii) a[%d]=%d, b[%d]=%d, c[%d]=%d", - i, a[i], i, b[i], i, c[i]) - } else { - t.Errorf("(iii) b[%d]=%d, c[%d]=%d", - i, b[i], i, c[i]) - } + t.Errorf("(ii) b[%d]=%d, c[%d]=%d", i, b[i], i, c[i]) } else { t.Logf("elem %d is okay\n", i) } diff --git a/src/pkg/reflect/value.go b/src/pkg/reflect/value.go index 2c2158a3c..3abe13e04 100644 --- a/src/pkg/reflect/value.go +++ b/src/pkg/reflect/value.go @@ -173,7 +173,7 @@ type nonEmptyInterface struct { // Regarding the implementation of Value: // // The Internal interface is a true interface value in the Go sense, -// but it also serves as a (type, address) pair in whcih one cannot +// but it also serves as a (type, address) pair in which one cannot // be changed separately from the other. That is, it serves as a way // to prevent unsafe mutations of the Internal state even though // we cannot (yet?) hide the field while preserving the ability for diff --git a/src/pkg/regexp/all_test.go b/src/pkg/regexp/all_test.go index c7ee4c879..71edc4d18 100644 --- a/src/pkg/regexp/all_test.go +++ b/src/pkg/regexp/all_test.go @@ -356,7 +356,7 @@ func BenchmarkMatchClass(b *testing.B) { func BenchmarkMatchClass_InRange(b *testing.B) { b.StopTimer() - // 'b' is betwen 'a' and 'c', so the charclass + // 'b' is between 'a' and 'c', so the charclass // range checking is no help here. x := strings.Repeat("bbbb", 20) + "c" re := MustCompile("[ac]") diff --git a/src/pkg/rpc/client.go b/src/pkg/rpc/client.go index 8af4afcf6..a8e560cbe 100644 --- a/src/pkg/rpc/client.go +++ b/src/pkg/rpc/client.go @@ -216,7 +216,7 @@ func DialHTTPPath(network, address, path string) (*Client, os.Error) { // Require successful HTTP response // before switching to RPC protocol. - resp, err := http.ReadResponse(bufio.NewReader(conn), "CONNECT") + resp, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: "CONNECT"}) if err == nil && resp.Status == connected { return NewClient(conn), nil } diff --git a/src/pkg/runtime/386/memmove.s b/src/pkg/runtime/386/memmove.s index 38a0652b5..471553ba2 100644 --- a/src/pkg/runtime/386/memmove.s +++ b/src/pkg/runtime/386/memmove.s @@ -32,7 +32,6 @@ TEXT runtime·memmove(SB), 7, $0 /* * check and set for backwards - * should we look closer for overlap? */ CMPL SI, DI JLS back @@ -40,6 +39,7 @@ TEXT runtime·memmove(SB), 7, $0 /* * forward copy loop */ +forward: MOVL BX, CX SHRL $2, CX ANDL $3, BX @@ -51,10 +51,18 @@ TEXT runtime·memmove(SB), 7, $0 MOVL to+0(FP),AX RET /* + * check overlap + */ +back: + MOVL SI, CX + ADDL BX, CX + CMPL CX, DI + JLS forward +/* * whole thing backwards has * adjusted addresses */ -back: + ADDL BX, DI ADDL BX, SI STD diff --git a/src/pkg/runtime/amd64/memmove.s b/src/pkg/runtime/amd64/memmove.s index 9966b0ba7..fc9573f72 100644 --- a/src/pkg/runtime/amd64/memmove.s +++ b/src/pkg/runtime/amd64/memmove.s @@ -33,7 +33,6 @@ TEXT runtime·memmove(SB), 7, $0 /* * check and set for backwards - * should we look closer for overlap? */ CMPQ SI, DI JLS back @@ -41,6 +40,7 @@ TEXT runtime·memmove(SB), 7, $0 /* * forward copy loop */ +forward: MOVQ BX, CX SHRQ $3, CX ANDQ $7, BX @@ -51,11 +51,19 @@ TEXT runtime·memmove(SB), 7, $0 MOVQ to+0(FP),AX RET +back: +/* + * check overlap + */ + MOVQ SI, CX + ADDQ BX, CX + CMPQ CX, DI + JLS forward + /* * whole thing backwards has * adjusted addresses */ -back: ADDQ BX, DI ADDQ BX, SI STD diff --git a/src/pkg/runtime/append_test.go b/src/pkg/runtime/append_test.go new file mode 100644 index 000000000..75a635306 --- /dev/null +++ b/src/pkg/runtime/append_test.go @@ -0,0 +1,51 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package runtime_test + +import "testing" + +const N = 20 + +func BenchmarkAppend(b *testing.B) { + b.StopTimer() + x := make([]int, 0, N) + b.StartTimer() + for i := 0; i < b.N; i++ { + x = x[0:0] + for j := 0; j < N; j++ { + x = append(x, j) + } + } +} + +func BenchmarkAppendSpecialCase(b *testing.B) { + b.StopTimer() + x := make([]int, 0, N) + b.StartTimer() + for i := 0; i < b.N; i++ { + x = x[0:0] + for j := 0; j < N; j++ { + if len(x) < cap(x) { + x = x[:len(x)+1] + x[len(x)-1] = j + } else { + x = append(x, j) + } + } + } +} + +var x = make([]int, 0, 10) + +func f() int { + x[:1][0] = 3 + return 2 +} + +func TestSideEffectOrder(t *testing.T) { + x = append(x, 1, f()) + if x[0] != 1 || x[1] != 2 { + t.Error("append failed: ", x[0], x[1]) + } +} diff --git a/src/pkg/runtime/arm/closure.c b/src/pkg/runtime/arm/closure.c index 3aca3a42d..36a93bc53 100644 --- a/src/pkg/runtime/arm/closure.c +++ b/src/pkg/runtime/arm/closure.c @@ -43,7 +43,7 @@ vars: WORD arg0 WORD arg2 */ -extern void cacheflush(byte* start, byte* end); +extern void runtime·cacheflush(byte* start, byte* end); #pragma textflag 7 void diff --git a/src/pkg/runtime/arm/softfloat.c b/src/pkg/runtime/arm/softfloat.c index f91a6fc09..0a071dada 100644 --- a/src/pkg/runtime/arm/softfloat.c +++ b/src/pkg/runtime/arm/softfloat.c @@ -15,6 +15,7 @@ #define FLAGS_V (1 << 28) void runtime·abort(void); +void math·sqrtGoC(uint64, uint64*); static uint32 trace = 0; @@ -357,6 +358,15 @@ stage3: // regd, regm are 4bit variables regd, regm, m->freghi[regd], m->freglo[regd]); break; + case 0xeeb10bc0: // D[regd] = sqrt D[regm] + math·sqrtGoC(getd(regm), &uval); + putd(regd, uval); + + if(trace) + runtime·printf("*** D[%d] = sqrt D[%d] %x-%x\n", + regd, regm, m->freghi[regd], m->freglo[regd]); + break; + case 0xeeb40bc0: // D[regd] :: D[regm] (CMPD) runtime·fcmp64c(getd(regd), getd(regm), &cmp, &nan); m->fflag = fstatus(nan, cmp); diff --git a/src/pkg/runtime/arm/vlop.s b/src/pkg/runtime/arm/vlop.s index 2c5d7ebe1..fc679f0ee 100644 --- a/src/pkg/runtime/arm/vlop.s +++ b/src/pkg/runtime/arm/vlop.s @@ -105,7 +105,7 @@ loop: /* * compare numerator to denominator - * if less, subtract and set quotent bit + * if less, subtract and set quotient bit */ CMP R(D), R(N) ORR.HS $1, R(Q) diff --git a/src/pkg/runtime/cgo/Makefile b/src/pkg/runtime/cgo/Makefile index 768fe80ac..f26da2c51 100644 --- a/src/pkg/runtime/cgo/Makefile +++ b/src/pkg/runtime/cgo/Makefile @@ -28,18 +28,20 @@ CGO_OFILES=\ $(GOOS)_$(GOARCH).o\ util.o\ -OFILES=\ - iscgo.$O\ - callbacks.$O\ - _cgo_import.$O\ - $(CGO_OFILES)\ - ifeq ($(GOOS),windows) CGO_LDFLAGS=-lm -mthreads else CGO_LDFLAGS=-lpthread +CGO_OFILES+=setenv.o\ + endif +OFILES=\ + iscgo.$O\ + callbacks.$O\ + _cgo_import.$O\ + $(CGO_OFILES)\ + ifeq ($(GOOS),freebsd) OFILES+=\ freebsd.$O\ diff --git a/src/pkg/runtime/cgo/setenv.c b/src/pkg/runtime/cgo/setenv.c new file mode 100644 index 000000000..c911b8392 --- /dev/null +++ b/src/pkg/runtime/cgo/setenv.c @@ -0,0 +1,16 @@ +// Copyright 20111 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "libcgo.h" + +#include <stdlib.h> + +/* Stub for calling setenv */ +static void +xlibcgo_setenv(char **arg) +{ + setenv(arg[0], arg[1], 1); +} + +void (*libcgo_setenv)(char**) = xlibcgo_setenv; diff --git a/src/pkg/runtime/closure_test.go b/src/pkg/runtime/closure_test.go new file mode 100644 index 000000000..ea65fbd5f --- /dev/null +++ b/src/pkg/runtime/closure_test.go @@ -0,0 +1,53 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package runtime_test + +import "testing" + +var s int + +func BenchmarkCallClosure(b *testing.B) { + for i := 0; i < b.N; i++ { + s += func(ii int) int { return 2 * ii }(i) + } +} + +func BenchmarkCallClosure1(b *testing.B) { + for i := 0; i < b.N; i++ { + j := i + s += func(ii int) int { return 2*ii + j }(i) + } +} + +var ss *int + +func BenchmarkCallClosure2(b *testing.B) { + for i := 0; i < b.N; i++ { + j := i + s += func() int { + ss = &j + return 2 + }() + } +} + +func addr1(x int) *int { + return func() *int { return &x }() +} + +func BenchmarkCallClosure3(b *testing.B) { + for i := 0; i < b.N; i++ { + ss = addr1(i) + } +} + +func addr2() (x int, p *int) { + return 0, func() *int { return &x }() +} + +func BenchmarkCallClosure4(b *testing.B) { + for i := 0; i < b.N; i++ { + _, ss = addr2() + } +} diff --git a/src/pkg/runtime/linux/386/sys.s b/src/pkg/runtime/linux/386/sys.s index 868a0d901..e8b423324 100644 --- a/src/pkg/runtime/linux/386/sys.s +++ b/src/pkg/runtime/linux/386/sys.s @@ -47,6 +47,14 @@ TEXT runtime·setitimer(SB),7,$0-24 INT $0x80 RET +TEXT runtime·mincore(SB),7,$0-24 + MOVL $218, AX // syscall - mincore + MOVL 4(SP), BX + MOVL 8(SP), CX + MOVL 12(SP), DX + INT $0x80 + RET + TEXT runtime·gettime(SB), 7, $32 MOVL $78, AX // syscall - gettimeofday LEAL 8(SP), BX diff --git a/src/pkg/runtime/linux/amd64/sys.s b/src/pkg/runtime/linux/amd64/sys.s index eadd30005..66fdab208 100644 --- a/src/pkg/runtime/linux/amd64/sys.s +++ b/src/pkg/runtime/linux/amd64/sys.s @@ -53,6 +53,14 @@ TEXT runtime·setitimer(SB),7,$0-24 SYSCALL RET +TEXT runtime·mincore(SB),7,$0-24 + MOVQ 8(SP), DI + MOVQ 16(SP), SI + MOVQ 24(SP), DX + MOVL $27, AX // syscall entry + SYSCALL + RET + TEXT runtime·gettime(SB), 7, $32 LEAQ 8(SP), DI MOVQ $0, SI diff --git a/src/pkg/runtime/linux/arm/sys.s b/src/pkg/runtime/linux/arm/sys.s index 2b5365bd8..ab5349822 100644 --- a/src/pkg/runtime/linux/arm/sys.s +++ b/src/pkg/runtime/linux/arm/sys.s @@ -26,6 +26,7 @@ #define SYS_exit_group (SYS_BASE + 248) #define SYS_munmap (SYS_BASE + 91) #define SYS_setitimer (SYS_BASE + 104) +#define SYS_mincore (SYS_BASE + 219) #define SYS_gettid (SYS_BASE + 224) #define SYS_tkill (SYS_BASE + 238) @@ -91,6 +92,14 @@ TEXT runtime·setitimer(SB),7,$0 SWI $0 RET +TEXT runtime·mincore(SB),7,$0 + MOVW 0(FP), R0 + MOVW 4(FP), R1 + MOVW 8(FP), R2 + MOVW $SYS_mincore, R7 + SWI $0 + RET + TEXT runtime·gettime(SB),7,$32 /* dummy version - return 0,0 */ MOVW $0, R1 diff --git a/src/pkg/runtime/linux/mem.c b/src/pkg/runtime/linux/mem.c index ce1a8aa70..38ca7e4a0 100644 --- a/src/pkg/runtime/linux/mem.c +++ b/src/pkg/runtime/linux/mem.c @@ -3,6 +3,30 @@ #include "os.h" #include "malloc.h" +enum +{ + ENOMEM = 12, +}; + +static int32 +addrspace_free(void *v, uintptr n) +{ + uintptr page_size = 4096; + uintptr off; + int8 one_byte; + + for(off = 0; off < n; off += page_size) { + int32 errval = runtime·mincore((int8 *)v + off, page_size, (void *)&one_byte); + // errval is 0 if success, or -(error_code) if error. + if (errval == 0 || errval != -ENOMEM) + return 0; + } + USED(v); + USED(n); + return 1; +} + + void* runtime·SysAlloc(uintptr n) { @@ -39,20 +63,21 @@ runtime·SysFree(void *v, uintptr n) void* runtime·SysReserve(void *v, uintptr n) { + void *p; + // On 64-bit, people with ulimit -v set complain if we reserve too // much address space. Instead, assume that the reservation is okay // and check the assumption in SysMap. if(sizeof(void*) == 8) return v; - return runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); + p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); + if(p < (void*)4096) { + return nil; + } + return p; } -enum -{ - ENOMEM = 12, -}; - void runtime·SysMap(void *v, uintptr n) { @@ -63,6 +88,13 @@ runtime·SysMap(void *v, uintptr n) // On 64-bit, we don't actually have v reserved, so tread carefully. if(sizeof(void*) == 8) { p = runtime·mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_PRIVATE, -1, 0); + if(p != v && addrspace_free(v, n)) { + // On some systems, mmap ignores v without + // MAP_FIXED, so retry if the address space is free. + p = runtime·mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); + } + if(p == (void*)ENOMEM) + runtime·throw("runtime: out of memory"); if(p != v) { runtime·printf("runtime: address space conflict: map(%p) = %p\n", v, p); runtime·throw("runtime: address space conflict"); @@ -71,7 +103,7 @@ runtime·SysMap(void *v, uintptr n) } p = runtime·mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)-ENOMEM) + if(p == (void*)ENOMEM) runtime·throw("runtime: out of memory"); if(p != v) runtime·throw("runtime: cannot map pages in arena address space"); diff --git a/src/pkg/runtime/linux/thread.c b/src/pkg/runtime/linux/thread.c index 7166b0ef2..7c7ca7b4e 100644 --- a/src/pkg/runtime/linux/thread.c +++ b/src/pkg/runtime/linux/thread.c @@ -116,7 +116,7 @@ again: // // We only really care that (v&1) == 1 (the lock is held), // and in fact there is a futex variant that could - // accomodate that check, but let's not get carried away.) + // accommodate that check, but let's not get carried away.) futexsleep(&l->key, v+2); // We're awake: remove ourselves from the count. @@ -252,10 +252,10 @@ runtime·newosproc(M *m, G *g, void *stk, void (*fn)(void)) stk, m, g, fn, runtime·clone, m->id, m->tls[0], &m); } - ret = runtime·clone(flags, stk, m, g, fn); - - if(ret < 0) - *(int32*)123 = 123; + if((ret = runtime·clone(flags, stk, m, g, fn)) < 0) { + runtime·printf("runtime: failed to create new OS thread (have %d already; errno=%d)\n", runtime·mcount(), -ret); + runtime·throw("runtime.newosproc"); + } } void diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc index 1f2d6da40..c55be9772 100644 --- a/src/pkg/runtime/malloc.goc +++ b/src/pkg/runtime/malloc.goc @@ -487,7 +487,7 @@ func SetFinalizer(obj Eface, finalizer Eface) { nret = (nret + sizeof(void*)-1) & ~(sizeof(void*)-1); if(runtime·getfinalizer(obj.data, 0)) { - runtime·printf("runtime.SetFinalizer: finalizer already set"); + runtime·printf("runtime.SetFinalizer: finalizer already set\n"); goto throw; } } diff --git a/src/pkg/runtime/mprof.goc b/src/pkg/runtime/mprof.goc index aae3d183f..517f96a31 100644 --- a/src/pkg/runtime/mprof.goc +++ b/src/pkg/runtime/mprof.goc @@ -113,7 +113,7 @@ static uintptr addrmem; // hashMultiplier is the bottom 32 bits of int((sqrt(5)-1)/2 * (1<<32)). // This is a good multiplier as suggested in CLR, Knuth. The hash // value is taken to be the top AddrHashBits bits of the bottom 32 bits -// of the muliplied value. +// of the multiplied value. enum { HashMultiplier = 2654435769U }; diff --git a/src/pkg/runtime/proc.c b/src/pkg/runtime/proc.c index 52784854f..c5af8b754 100644 --- a/src/pkg/runtime/proc.c +++ b/src/pkg/runtime/proc.c @@ -714,7 +714,7 @@ runtime·oldstack(void) goid = old.gobuf.g->goid; // fault if g is bad, before gogo if(old.free != 0) - runtime·stackfree(g1->stackguard - StackGuard - StackSystem, old.free); + runtime·stackfree(g1->stackguard - StackGuard, old.free); g1->stackbase = old.stackbase; g1->stackguard = old.stackguard; @@ -756,7 +756,7 @@ runtime·newstack(void) // the new Stktop* is necessary to unwind, but // we don't need to create a new segment. top = (Stktop*)(m->morebuf.sp - sizeof(*top)); - stk = g1->stackguard - StackGuard - StackSystem; + stk = g1->stackguard - StackGuard; free = 0; } else { // allocate new segment. @@ -785,7 +785,7 @@ runtime·newstack(void) g1->ispanic = false; g1->stackbase = (byte*)top; - g1->stackguard = stk + StackGuard + StackSystem; + g1->stackguard = stk + StackGuard; sp = (byte*)top; if(argsize > 0) { @@ -834,7 +834,7 @@ runtime·malg(int32 stacksize) g->param = nil; } newg->stack0 = stk; - newg->stackguard = stk + StackSystem + StackGuard; + newg->stackguard = stk + StackGuard; newg->stackbase = stk + StackSystem + stacksize - sizeof(Stktop); runtime·memclr(newg->stackbase, sizeof(Stktop)); } @@ -880,7 +880,7 @@ runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret, void *callerpc) if((newg = gfget()) != nil){ newg->status = Gwaiting; - if(newg->stackguard - StackGuard - StackSystem != newg->stack0) + if(newg->stackguard - StackGuard != newg->stack0) runtime·throw("invalid stack in newg"); } else { newg = runtime·malg(StackMin); @@ -1165,7 +1165,7 @@ nomatch: static void gfput(G *g) { - if(g->stackguard - StackGuard - StackSystem != g->stack0) + if(g->stackguard - StackGuard != g->stack0) runtime·throw("invalid stack in gfput"); g->schedlink = runtime·sched.gfree; runtime·sched.gfree = g; @@ -1343,3 +1343,26 @@ runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz) if(hz != 0) runtime·resetcpuprofiler(hz); } + +void (*libcgo_setenv)(byte**); + +void +os·setenv_c(String k, String v) +{ + byte *arg[2]; + + if(libcgo_setenv == nil) + return; + + arg[0] = runtime·malloc(k.len + 1); + runtime·mcpy(arg[0], k.str, k.len); + arg[0][k.len] = 0; + + arg[1] = runtime·malloc(v.len + 1); + runtime·mcpy(arg[1], v.str, v.len); + arg[1][v.len] = 0; + + runtime·asmcgocall(libcgo_setenv, arg); + runtime·free(arg[0]); + runtime·free(arg[1]); +} diff --git a/src/pkg/runtime/proc_test.go b/src/pkg/runtime/proc_test.go index a15b2d80a..cac4f9eea 100644 --- a/src/pkg/runtime/proc_test.go +++ b/src/pkg/runtime/proc_test.go @@ -24,20 +24,23 @@ func TestStopTheWorldDeadlock(t *testing.T) { t.Logf("skipping during short test") return } - runtime.GOMAXPROCS(3) - compl := make(chan int, 1) + maxprocs := runtime.GOMAXPROCS(3) + compl := make(chan bool, 2) go func() { for i := 0; i != 1000; i += 1 { runtime.GC() } - compl <- 0 + compl <- true }() go func() { for i := 0; i != 1000; i += 1 { runtime.GOMAXPROCS(3) } + compl <- true }() go perpetuumMobile() <-compl + <-compl stop <- true + runtime.GOMAXPROCS(maxprocs) } diff --git a/src/pkg/runtime/runtime-gdb.py b/src/pkg/runtime/runtime-gdb.py index 3f767fbdd..a96f3f382 100644 --- a/src/pkg/runtime/runtime-gdb.py +++ b/src/pkg/runtime/runtime-gdb.py @@ -2,7 +2,7 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -"""GDB Pretty printers and convencience functions for Go's runtime structures. +"""GDB Pretty printers and convenience functions for Go's runtime structures. This script is loaded by GDB when it finds a .debug_gdb_scripts section in the compiled binary. The [68]l linkers emit this with a @@ -393,7 +393,7 @@ class GoIfaceCmd(gdb.Command): # so Itype will start with a commontype which has kind = interface # -# Register all convience functions and CLI commands +# Register all convenience functions and CLI commands # for k in vars().values(): if hasattr(k, 'invoke'): diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h index f9b404e15..f3ccff1bc 100644 --- a/src/pkg/runtime/runtime.h +++ b/src/pkg/runtime/runtime.h @@ -306,7 +306,7 @@ enum { /* * defined macros - * you need super-goru privilege + * you need super-gopher-guru privilege * to add this list. */ #define nelem(x) (sizeof(x)/sizeof((x)[0])) @@ -413,6 +413,7 @@ int32 runtime·gotraceback(void); void runtime·traceback(uint8 *pc, uint8 *sp, uint8 *lr, G* gp); void runtime·tracebackothers(G*); int32 runtime·write(int32, void*, int32); +int32 runtime·mincore(void*, uintptr, byte*); bool runtime·cas(uint32*, uint32, uint32); bool runtime·casp(void**, void*, void*); uint32 runtime·xadd(uint32 volatile*, int32); @@ -597,17 +598,3 @@ int32 runtime·chancap(Hchan*); void runtime·ifaceE2I(struct InterfaceType*, Eface, Iface*); -enum -{ - // StackSystem is a number of additional bytes to add - // to each stack below the usual guard area for OS-specific - // purposes like signal handling. - // TODO(rsc): This is only for Windows. Can't Windows use - // a separate exception stack like every other operating system? -#ifdef __WINDOWS__ - StackSystem = 2048, -#else - StackSystem = 0, -#endif -}; - diff --git a/src/pkg/runtime/slice.c b/src/pkg/runtime/slice.c index 1fee923e4..9146c177f 100644 --- a/src/pkg/runtime/slice.c +++ b/src/pkg/runtime/slice.c @@ -9,6 +9,8 @@ static int32 debug = 0; static void makeslice1(SliceType*, int32, int32, Slice*); +static void growslice1(SliceType*, Slice, int32, Slice *); +static void appendslice1(SliceType*, Slice, Slice, Slice*); void runtime·slicecopy(Slice to, Slice fm, uintptr width, int32 ret); // see also unsafe·NewArray @@ -46,22 +48,6 @@ makeslice1(SliceType *t, int32 len, int32 cap, Slice *ret) ret->array = runtime·mal(size); } -static void appendslice1(SliceType*, Slice, Slice, Slice*); - -// append(type *Type, n int, old []T, ...,) []T -#pragma textflag 7 -void -runtime·append(SliceType *t, int32 n, Slice old, ...) -{ - Slice sl; - Slice *ret; - - sl.len = n; - sl.array = (byte*)(&old+1); - ret = (Slice*)(sl.array + ((t->elem->size*n+sizeof(uintptr)-1) & ~(sizeof(uintptr)-1))); - appendslice1(t, old, sl, ret); -} - // appendslice(type *Type, x, y, []T) []T void runtime·appendslice(SliceType *t, Slice x, Slice y, Slice ret) @@ -72,36 +58,69 @@ runtime·appendslice(SliceType *t, Slice x, Slice y, Slice ret) static void appendslice1(SliceType *t, Slice x, Slice y, Slice *ret) { - Slice newx; int32 m; uintptr w; - if(x.len+y.len < x.len) + m = x.len+y.len; + + if(m < x.len) runtime·throw("append: slice overflow"); + if(m > x.cap) + growslice1(t, x, m, ret); + else + *ret = x; + w = t->elem->size; - if(x.len+y.len > x.cap) { - m = x.cap; - if(m == 0) - m = y.len; - else { - do { - if(x.len < 1024) - m += m; - else - m += m/4; - } while(m < x.len+y.len); - } - makeslice1(t, x.len, m, &newx); - runtime·memmove(newx.array, x.array, x.len*w); - x = newx; + runtime·memmove(ret->array + ret->len*w, y.array, y.len*w); + ret->len += y.len; +} + +// growslice(type *Type, x, []T, n int64) []T +void +runtime·growslice(SliceType *t, Slice old, int64 n, Slice ret) +{ + int64 cap; + + if(n < 1) + runtime·panicstring("growslice: invalid n"); + + cap = old.cap + n; + + if((int32)cap != cap || cap > ((uintptr)-1) / t->elem->size) + runtime·panicstring("growslice: cap out of range"); + + growslice1(t, old, cap, &ret); + + FLUSH(&ret); + + if(debug) { + runtime·printf("growslice(%S,", *t->string); + runtime·printslice(old); + runtime·printf(", new cap=%D) =", cap); + runtime·printslice(ret); } - runtime·memmove(x.array+x.len*w, y.array, y.len*w); - x.len += y.len; - *ret = x; } +static void +growslice1(SliceType *t, Slice x, int32 newcap, Slice *ret) +{ + int32 m; + m = x.cap; + if(m == 0) + m = newcap; + else { + do { + if(x.len < 1024) + m += m; + else + m += m/4; + } while(m < newcap); + } + makeslice1(t, x.len, m, ret); + runtime·memmove(ret->array, x.array, ret->len * t->elem->size); +} // sliceslice(old []any, lb uint64, hb uint64, width uint64) (ary []any); void diff --git a/src/pkg/runtime/softfloat64.go b/src/pkg/runtime/softfloat64.go index d9bbe5def..e0c3b7b73 100644 --- a/src/pkg/runtime/softfloat64.go +++ b/src/pkg/runtime/softfloat64.go @@ -11,7 +11,7 @@ package runtime const ( mantbits64 uint = 52 expbits64 uint = 11 - bias64 = -1<<(expbits64-1) + 1 + bias64 = -1<<(expbits64-1) + 1 nan64 uint64 = (1<<expbits64-1)<<mantbits64 + 1 inf64 uint64 = (1<<expbits64 - 1) << mantbits64 @@ -19,7 +19,7 @@ const ( mantbits32 uint = 23 expbits32 uint = 8 - bias32 = -1<<(expbits32-1) + 1 + bias32 = -1<<(expbits32-1) + 1 nan32 uint32 = (1<<expbits32-1)<<mantbits32 + 1 inf32 uint32 = (1<<expbits32 - 1) << mantbits32 diff --git a/src/pkg/runtime/stack.h b/src/pkg/runtime/stack.h index ebf0462b5..2b6b0e387 100644 --- a/src/pkg/runtime/stack.h +++ b/src/pkg/runtime/stack.h @@ -53,6 +53,16 @@ functions to make sure that this limit cannot be violated. */ enum { + // StackSystem is a number of additional bytes to add + // to each stack below the usual guard area for OS-specific + // purposes like signal handling. Used on Windows because + // it does not use a separate stack. +#ifdef __WINDOWS__ + StackSystem = 2048, +#else + StackSystem = 0, +#endif + // The amount of extra stack to allocate beyond the size // needed for the single frame that triggered the split. StackExtra = 1024, @@ -73,7 +83,7 @@ enum { // The stack guard is a pointer this many bytes above the // bottom of the stack. - StackGuard = 256, + StackGuard = 256 + StackSystem, // After a stack split check the SP is allowed to be this // many bytes below the stack guard. This saves an instruction @@ -82,5 +92,5 @@ enum { // The maximum number of bytes that a chain of NOSPLIT // functions can use. - StackLimit = StackGuard - StackSmall, + StackLimit = StackGuard - StackSystem - StackSmall, }; diff --git a/src/pkg/runtime/windows/386/sys.s b/src/pkg/runtime/windows/386/sys.s index 15f7f95b8..d38405075 100644 --- a/src/pkg/runtime/windows/386/sys.s +++ b/src/pkg/runtime/windows/386/sys.s @@ -59,15 +59,21 @@ TEXT runtime·setlasterror(SB),7,$0 TEXT runtime·sigtramp(SB),7,$0 PUSHL BP // cdecl + PUSHL BX + PUSHL SI + PUSHL DI PUSHL 0(FS) CALL runtime·sigtramp1(SB) POPL 0(FS) + POPL DI + POPL SI + POPL BX POPL BP RET -TEXT runtime·sigtramp1(SB),0,$16-28 +TEXT runtime·sigtramp1(SB),0,$16-40 // unwinding? - MOVL info+12(FP), BX + MOVL info+24(FP), BX MOVL 4(BX), CX // exception flags ANDL $6, CX MOVL $1, AX @@ -75,15 +81,15 @@ TEXT runtime·sigtramp1(SB),0,$16-28 // place ourselves at the top of the SEH chain to // ensure SEH frames lie within thread stack bounds - MOVL frame+16(FP), CX // our SEH frame + MOVL frame+28(FP), CX // our SEH frame MOVL CX, 0(FS) // copy arguments for call to sighandler MOVL BX, 0(SP) MOVL CX, 4(SP) - MOVL context+20(FP), BX + MOVL context+32(FP), BX MOVL BX, 8(SP) - MOVL dispatcher+24(FP), BX + MOVL dispatcher+36(FP), BX MOVL BX, 12(SP) CALL runtime·sighandler(SB) diff --git a/src/pkg/runtime/windows/thread.c b/src/pkg/runtime/windows/thread.c index 2ce92dcfb..81ad68033 100644 --- a/src/pkg/runtime/windows/thread.c +++ b/src/pkg/runtime/windows/thread.c @@ -324,13 +324,31 @@ runtime·ctrlhandler1(uint32 type) return 0; } +// Will keep all callbacks in a linked list, so they don't get garbage collected. +typedef struct Callback Callback; +struct Callback { + Callback* link; + void* gobody; + byte asmbody; +}; + +typedef struct Callbacks Callbacks; +struct Callbacks { + Lock; + Callback* link; + int32 n; +}; + +static Callbacks cbs; + // Call back from windows dll into go. byte * runtime·compilecallback(Eface fn, bool cleanstack) { Func *f; int32 argsize, n; - byte *ret, *p; + byte *p; + Callback *c; if(fn.type->kind != KindFunc) runtime·panicstring("not a function"); @@ -348,7 +366,23 @@ runtime·compilecallback(Eface fn, bool cleanstack) if(cleanstack) n += 2; // ... argsize - ret = p = runtime·mal(n); + runtime·lock(&cbs); + for(c = cbs.link; c != nil; c = c->link) { + if(c->gobody == fn.data) { + runtime·unlock(&cbs); + return &c->asmbody; + } + } + if(cbs.n >= 2000) + runtime·throw("too many callback functions"); + c = runtime·mal(sizeof *c + n); + c->gobody = fn.data; + c->link = cbs.link; + cbs.link = c; + cbs.n++; + runtime·unlock(&cbs); + + p = &c->asmbody; // MOVL fn, AX *p++ = 0xb8; @@ -376,7 +410,7 @@ runtime·compilecallback(Eface fn, bool cleanstack) } else *p = 0xc3; - return ret; + return &c->asmbody; } void diff --git a/src/pkg/smtp/smtp.go b/src/pkg/smtp/smtp.go index 3f89af147..d716df56b 100644 --- a/src/pkg/smtp/smtp.go +++ b/src/pkg/smtp/smtp.go @@ -151,8 +151,7 @@ func (c *Client) Auth(a Auth) os.Error { var msg []byte switch code { case 334: - msg = make([]byte, encoding.DecodedLen(len(msg64))) - _, err = encoding.Decode(msg, []byte(msg64)) + msg, err = encoding.DecodeString(msg64) case 235: // the last message isn't base64 because it isn't a challenge msg = []byte(msg64) diff --git a/src/pkg/strconv/atob.go b/src/pkg/strconv/atob.go index 69fa2292a..98ce75079 100644 --- a/src/pkg/strconv/atob.go +++ b/src/pkg/strconv/atob.go @@ -7,8 +7,8 @@ package strconv import "os" // Atob returns the boolean value represented by the string. -// It accepts 1, t, T, TRUE, true, 0, f, F, FALSE, false. Any other value returns -// an error. +// It accepts 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False. +// Any other value returns an error. func Atob(str string) (value bool, err os.Error) { switch str { case "1", "t", "T", "true", "TRUE", "True": diff --git a/src/pkg/strconv/atob_test.go b/src/pkg/strconv/atob_test.go index 497df5b18..541e60d1e 100644 --- a/src/pkg/strconv/atob_test.go +++ b/src/pkg/strconv/atob_test.go @@ -24,11 +24,13 @@ var atobtests = []atobTest{ {"F", false, nil}, {"FALSE", false, nil}, {"false", false, nil}, + {"False", false, nil}, {"1", true, nil}, {"t", true, nil}, {"T", true, nil}, {"TRUE", true, nil}, {"true", true, nil}, + {"True", true, nil}, } func TestAtob(t *testing.T) { diff --git a/src/pkg/strconv/decimal.go b/src/pkg/strconv/decimal.go index 3a5cf1ba6..783065bfb 100644 --- a/src/pkg/strconv/decimal.go +++ b/src/pkg/strconv/decimal.go @@ -108,7 +108,7 @@ func newDecimal(i uint64) *decimal { } // Maximum shift that we can do in one pass without overflow. -// Signed int has 31 bits, and we have to be able to accomodate 9<<k. +// Signed int has 31 bits, and we have to be able to accommodate 9<<k. const maxShift = 27 // Binary shift right (* 2) by k bits. k <= maxShift to avoid overflow. diff --git a/src/pkg/strconv/quote.go b/src/pkg/strconv/quote.go index ed5889723..98b19d3a2 100644 --- a/src/pkg/strconv/quote.go +++ b/src/pkg/strconv/quote.go @@ -14,63 +14,107 @@ import ( const lowerhex = "0123456789abcdef" -// Quote returns a double-quoted Go string literal -// representing s. The returned string s uses Go escape -// sequences (\t, \n, \xFF, \u0100) for control characters -// and non-ASCII characters. -func Quote(s string) string { +func quoteWith(s string, quote byte, ASCIIonly bool) string { var buf bytes.Buffer - buf.WriteByte('"') - for ; len(s) > 0; s = s[1:] { - switch c := s[0]; { - case c == '"': - buf.WriteString(`\"`) - case c == '\\': - buf.WriteString(`\\`) - case ' ' <= c && c <= '~': - buf.WriteString(string(c)) - case c == '\a': + buf.WriteByte(quote) + for width := 0; len(s) > 0; s = s[width:] { + rune := int(s[0]) + width = 1 + if rune >= utf8.RuneSelf { + rune, width = utf8.DecodeRuneInString(s) + } + if width == 1 && rune == utf8.RuneError { + goto printEscX + } + if rune == int(quote) || rune == '\\' { // always backslashed + buf.WriteByte('\\') + buf.WriteByte(byte(rune)) + continue + } + if ASCIIonly { + if rune <= unicode.MaxASCII && unicode.IsPrint(rune) { + buf.WriteRune(rune) + continue + } + } else if unicode.IsPrint(rune) { + buf.WriteRune(rune) + continue + } + switch rune { + case '\a': buf.WriteString(`\a`) - case c == '\b': + case '\b': buf.WriteString(`\b`) - case c == '\f': + case '\f': buf.WriteString(`\f`) - case c == '\n': + case '\n': buf.WriteString(`\n`) - case c == '\r': + case '\r': buf.WriteString(`\r`) - case c == '\t': + case '\t': buf.WriteString(`\t`) - case c == '\v': + case '\v': buf.WriteString(`\v`) - - case c >= utf8.RuneSelf && utf8.FullRuneInString(s): - r, size := utf8.DecodeRuneInString(s) - if r == utf8.RuneError && size == 1 { - goto EscX - } - s = s[size-1:] // next iteration will slice off 1 more - if r < 0x10000 { + default: + switch { + case rune < ' ': + printEscX: + buf.WriteString(`\x`) + buf.WriteByte(lowerhex[s[0]>>4]) + buf.WriteByte(lowerhex[s[0]&0xF]) + case rune > unicode.MaxRune: + rune = 0xFFFD + fallthrough + case rune < 0x10000: buf.WriteString(`\u`) - for j := uint(0); j < 4; j++ { - buf.WriteByte(lowerhex[(r>>(12-4*j))&0xF]) + for s := 12; s >= 0; s -= 4 { + buf.WriteByte(lowerhex[rune>>uint(s)&0xF]) } - } else { + default: buf.WriteString(`\U`) - for j := uint(0); j < 8; j++ { - buf.WriteByte(lowerhex[(r>>(28-4*j))&0xF]) + for s := 28; s >= 0; s -= 4 { + buf.WriteByte(lowerhex[rune>>uint(s)&0xF]) } } - - default: - EscX: - buf.WriteString(`\x`) - buf.WriteByte(lowerhex[c>>4]) - buf.WriteByte(lowerhex[c&0xF]) } } - buf.WriteByte('"') + buf.WriteByte(quote) return buf.String() + +} + +// Quote returns a double-quoted Go string literal representing s. The +// returned string uses Go escape sequences (\t, \n, \xFF, \u0100) for +// control characters and non-printable characters as defined by +// unicode.IsPrint. +func Quote(s string) string { + return quoteWith(s, '"', false) +} + +// QuoteToASCII returns a double-quoted Go string literal representing s. +// The returned string uses Go escape sequences (\t, \n, \xFF, \u0100) for +// non-ASCII characters and non-printable characters as defined by +// unicode.IsPrint. +func QuoteToASCII(s string) string { + return quoteWith(s, '"', true) +} + +// QuoteRune returns a single-quoted Go character literal representing the +// rune. The returned string uses Go escape sequences (\t, \n, \xFF, \u0100) +// for control characters and non-printable characters as defined by +// unicode.IsPrint. +func QuoteRune(rune int) string { + // TODO: avoid the allocation here. + return quoteWith(string(rune), '\'', false) +} + +// QuoteRuneToASCII returns a single-quoted Go character literal representing +// the rune. The returned string uses Go escape sequences (\t, \n, \xFF, +// \u0100) for non-ASCII characters and non-printable characters as defined +// by unicode.IsPrint. +func QuoteRuneToASCII(rune int) string { + // TODO: avoid the allocation here. + return quoteWith(string(rune), '\'', true) } // CanBackquote returns whether the string s would be diff --git a/src/pkg/strconv/quote_test.go b/src/pkg/strconv/quote_test.go index 1235fcb9a..4d615db44 100644 --- a/src/pkg/strconv/quote_test.go +++ b/src/pkg/strconv/quote_test.go @@ -11,28 +11,70 @@ import ( ) type quoteTest struct { - in string - out string + in string + out string + ascii string } var quotetests = []quoteTest{ - {"\a\b\f\r\n\t\v", `"\a\b\f\r\n\t\v"`}, - {"\\", `"\\"`}, - {"abc\xffdef", `"abc\xffdef"`}, - {"\u263a", `"\u263a"`}, - {"\U0010ffff", `"\U0010ffff"`}, - {"\x04", `"\x04"`}, + {"\a\b\f\r\n\t\v", `"\a\b\f\r\n\t\v"`, `"\a\b\f\r\n\t\v"`}, + {"\\", `"\\"`, `"\\"`}, + {"abc\xffdef", `"abc\xffdef"`, `"abc\xffdef"`}, + {"\u263a", `"☺"`, `"\u263a"`}, + {"\U0010ffff", `"\U0010ffff"`, `"\U0010ffff"`}, + {"\x04", `"\x04"`, `"\x04"`}, } func TestQuote(t *testing.T) { - for i := 0; i < len(quotetests); i++ { - tt := quotetests[i] + for _, tt := range quotetests { if out := Quote(tt.in); out != tt.out { t.Errorf("Quote(%s) = %s, want %s", tt.in, out, tt.out) } } } +func TestQuoteToASCII(t *testing.T) { + for _, tt := range quotetests { + if out := QuoteToASCII(tt.in); out != tt.ascii { + t.Errorf("QuoteToASCII(%s) = %s, want %s", tt.in, out, tt.ascii) + } + } +} + +type quoteRuneTest struct { + in int + out string + ascii string +} + +var quoterunetests = []quoteRuneTest{ + {'a', `'a'`, `'a'`}, + {'\a', `'\a'`, `'\a'`}, + {'\\', `'\\'`, `'\\'`}, + {0xFF, `'ÿ'`, `'\u00ff'`}, + {0x263a, `'☺'`, `'\u263a'`}, + {0xfffd, `'�'`, `'\ufffd'`}, + {0x0010ffff, `'\U0010ffff'`, `'\U0010ffff'`}, + {0x0010ffff + 1, `'�'`, `'\ufffd'`}, + {0x04, `'\x04'`, `'\x04'`}, +} + +func TestQuoteRune(t *testing.T) { + for _, tt := range quoterunetests { + if out := QuoteRune(tt.in); out != tt.out { + t.Errorf("QuoteRune(%U) = %s, want %s", tt.in, out, tt.out) + } + } +} + +func TestQuoteRuneToASCII(t *testing.T) { + for _, tt := range quoterunetests { + if out := QuoteRuneToASCII(tt.in); out != tt.ascii { + t.Errorf("QuoteRuneToASCII(%U) = %s, want %s", tt.in, out, tt.ascii) + } + } +} + type canBackquoteTest struct { in string out bool @@ -80,15 +122,19 @@ var canbackquotetests = []canBackquoteTest{ } func TestCanBackquote(t *testing.T) { - for i := 0; i < len(canbackquotetests); i++ { - tt := canbackquotetests[i] + for _, tt := range canbackquotetests { if out := CanBackquote(tt.in); out != tt.out { t.Errorf("CanBackquote(%q) = %v, want %v", tt.in, out, tt.out) } } } -var unquotetests = []quoteTest{ +type unQuoteTest struct { + in string + out string +} + +var unquotetests = []unQuoteTest{ {`""`, ""}, {`"a"`, "a"}, {`"abc"`, "abc"}, @@ -146,23 +192,20 @@ var misquoted = []string{ } func TestUnquote(t *testing.T) { - for i := 0; i < len(unquotetests); i++ { - tt := unquotetests[i] + for _, tt := range unquotetests { if out, err := Unquote(tt.in); err != nil && out != tt.out { t.Errorf("Unquote(%#q) = %q, %v want %q, nil", tt.in, out, err, tt.out) } } // run the quote tests too, backward - for i := 0; i < len(quotetests); i++ { - tt := quotetests[i] + for _, tt := range quotetests { if in, err := Unquote(tt.out); in != tt.in { t.Errorf("Unquote(%#q) = %q, %v, want %q, nil", tt.out, in, err, tt.in) } } - for i := 0; i < len(misquoted); i++ { - s := misquoted[i] + for _, s := range misquoted { if out, err := Unquote(s); out != "" || err != os.EINVAL { t.Errorf("Unquote(%#q) = %q, %v want %q, %v", s, out, err, "", os.EINVAL) } diff --git a/src/pkg/strings/reader.go b/src/pkg/strings/reader.go index 914faa003..10b0278e1 100644 --- a/src/pkg/strings/reader.go +++ b/src/pkg/strings/reader.go @@ -9,53 +9,84 @@ import ( "utf8" ) -// A Reader satisfies calls to Read, ReadByte, and ReadRune by -// reading from a string. -type Reader string +// A Reader implements the io.Reader, io.ByteScanner, and +// io.RuneScanner interfaces by reading from a string. +type Reader struct { + s string + i int // current reading index + prevRune int // index of previous rune; or < 0 +} + +// Len returns the number of bytes of the unread portion of the +// string. +func (r *Reader) Len() int { + return len(r.s) - r.i +} func (r *Reader) Read(b []byte) (n int, err os.Error) { - s := *r - if len(s) == 0 { + if r.i >= len(r.s) { return 0, os.EOF } - for n < len(s) && n < len(b) { - b[n] = s[n] - n++ - } - *r = s[n:] + n = copy(b, r.s[r.i:]) + r.i += n + r.prevRune = -1 return } func (r *Reader) ReadByte() (b byte, err os.Error) { - s := *r - if len(s) == 0 { + if r.i >= len(r.s) { return 0, os.EOF } - b = s[0] - *r = s[1:] + b = r.s[r.i] + r.i++ + r.prevRune = -1 return } + +// UnreadByte moves the reading position back by one byte. +// It is an error to call UnreadByte if nothing has been +// read yet. +func (r *Reader) UnreadByte() os.Error { + if r.i <= 0 { + return os.ErrorString("strings.Reader: at beginning of string") + } + r.i-- + r.prevRune = -1 + return nil +} + // ReadRune reads and returns the next UTF-8-encoded // Unicode code point from the buffer. // If no bytes are available, the error returned is os.EOF. // If the bytes are an erroneous UTF-8 encoding, it // consumes one byte and returns U+FFFD, 1. func (r *Reader) ReadRune() (rune int, size int, err os.Error) { - s := *r - if len(s) == 0 { + if r.i >= len(r.s) { return 0, 0, os.EOF } - c := s[0] - if c < utf8.RuneSelf { - *r = s[1:] + r.prevRune = r.i + if c := r.s[r.i]; c < utf8.RuneSelf { + r.i++ return int(c), 1, nil } - rune, size = utf8.DecodeRuneInString(string(s)) - *r = s[size:] + rune, size = utf8.DecodeRuneInString(r.s[r.i:]) + r.i += size return } +// UnreadRune causes the next call to ReadRune to return the same rune +// as the previous call to ReadRune. +// The last method called on r must have been ReadRune. +func (r *Reader) UnreadRune() os.Error { + if r.prevRune < 0 { + return os.ErrorString("strings.Reader: previous operation was not ReadRune") + } + r.i = r.prevRune + r.prevRune = -1 + return nil +} + // NewReader returns a new Reader reading from s. // It is similar to bytes.NewBufferString but more efficient and read-only. -func NewReader(s string) *Reader { return (*Reader)(&s) } +func NewReader(s string) *Reader { return &Reader{s, 0, -1} } diff --git a/src/pkg/strings/strings_test.go b/src/pkg/strings/strings_test.go index c45b1485d..a1a635ddd 100644 --- a/src/pkg/strings/strings_test.go +++ b/src/pkg/strings/strings_test.go @@ -5,6 +5,7 @@ package strings_test import ( + "bytes" "os" "reflect" "strconv" @@ -751,13 +752,56 @@ func TestRunes(t *testing.T) { } } +func TestReadByte(t *testing.T) { + testStrings := []string{"", abcd, faces, commas} + for _, s := range testStrings { + reader := NewReader(s) + if e := reader.UnreadByte(); e == nil { + t.Errorf("Unreading %q at beginning: expected error", s) + } + var res bytes.Buffer + for { + b, e := reader.ReadByte() + if e == os.EOF { + break + } + if e != nil { + t.Errorf("Reading %q: %s", s, e) + break + } + res.WriteByte(b) + // unread and read again + e = reader.UnreadByte() + if e != nil { + t.Errorf("Unreading %q: %s", s, e) + break + } + b1, e := reader.ReadByte() + if e != nil { + t.Errorf("Reading %q after unreading: %s", s, e) + break + } + if b1 != b { + t.Errorf("Reading %q after unreading: want byte %q, got %q", s, b, b1) + break + } + } + if res.String() != s { + t.Errorf("Reader(%q).ReadByte() produced %q", s, res.String()) + } + } +} + func TestReadRune(t *testing.T) { testStrings := []string{"", abcd, faces, commas} for _, s := range testStrings { reader := NewReader(s) + if e := reader.UnreadRune(); e == nil { + t.Errorf("Unreading %q at beginning: expected error", s) + } res := "" for { - r, _, e := reader.ReadRune() + r, z, e := reader.ReadRune() if e == os.EOF { break } @@ -766,6 +810,25 @@ func TestReadRune(t *testing.T) { break } res += string(r) + // unread and read again + e = reader.UnreadRune() + if e != nil { + t.Errorf("Unreading %q: %s", s, e) + break + } + r1, z1, e := reader.ReadRune() + if e != nil { + t.Errorf("Reading %q after unreading: %s", s, e) + break + } + if r1 != r { + t.Errorf("Reading %q after unreading: want rune %q, got %q", s, r, r1) + break + } + if z1 != z { + t.Errorf("Reading %q after unreading: want size %d, got %d", s, z, z1) + break + } } if res != s { t.Errorf("Reader(%q).ReadRune() produced %q", s, res) diff --git a/src/pkg/sync/atomic/asm_arm.s b/src/pkg/sync/atomic/asm_arm.s index 3363bbcf1..95e2f5be4 100644 --- a/src/pkg/sync/atomic/asm_arm.s +++ b/src/pkg/sync/atomic/asm_arm.s @@ -87,11 +87,14 @@ add64loop: // which will make uses of the 64-bit atomic operations loop forever. // If things are working, set okLDREXD to avoid future checks. // https://bugs.launchpad.net/qemu/+bug/670883. -TEXT check64<>(SB),7,$8 +TEXT check64<>(SB),7,$16 MOVW $10, R1 + // 8-aligned stack address scratch space. + MOVW $8(R13), R5 + AND $~7, R5 loop: - LDREXD (SP), R2 - STREXD R2, (SP), R0 + LDREXD (R5), R2 + STREXD R2, (R5), R0 CMP $0, R0 BEQ ok SUB $1, R1 diff --git a/src/pkg/sync/cond.go b/src/pkg/sync/cond.go index ea48f2e7a..75494b535 100644 --- a/src/pkg/sync/cond.go +++ b/src/pkg/sync/cond.go @@ -14,10 +14,26 @@ import "runtime" // which must be held when changing the condition and // when calling the Wait method. type Cond struct { - L Locker // held while observing or changing the condition - m Mutex // held to avoid internal races - waiters int // number of goroutines blocked on Wait - sema *uint32 + L Locker // held while observing or changing the condition + m Mutex // held to avoid internal races + + // We must be careful to make sure that when Signal + // releases a semaphore, the corresponding acquire is + // executed by a goroutine that was already waiting at + // the time of the call to Signal, not one that arrived later. + // To ensure this, we segment waiting goroutines into + // generations punctuated by calls to Signal. Each call to + // Signal begins another generation if there are no goroutines + // left in older generations for it to wake. Because of this + // optimization (only begin another generation if there + // are no older goroutines left), we only need to keep track + // of the two most recent generations, which we call old + // and new. + oldWaiters int // number of waiters in old generation... + oldSema *uint32 // ... waiting on this semaphore + + newWaiters int // number of waiters in new generation... + newSema *uint32 // ... waiting on this semaphore } // NewCond returns a new Cond with Locker l. @@ -42,11 +58,11 @@ func NewCond(l Locker) *Cond { // func (c *Cond) Wait() { c.m.Lock() - if c.sema == nil { - c.sema = new(uint32) + if c.newSema == nil { + c.newSema = new(uint32) } - s := c.sema - c.waiters++ + s := c.newSema + c.newWaiters++ c.m.Unlock() c.L.Unlock() runtime.Semacquire(s) @@ -59,9 +75,16 @@ func (c *Cond) Wait() { // during the call. func (c *Cond) Signal() { c.m.Lock() - if c.waiters > 0 { - c.waiters-- - runtime.Semrelease(c.sema) + if c.oldWaiters == 0 && c.newWaiters > 0 { + // Retire old generation; rename new to old. + c.oldWaiters = c.newWaiters + c.oldSema = c.newSema + c.newWaiters = 0 + c.newSema = nil + } + if c.oldWaiters > 0 { + c.oldWaiters-- + runtime.Semrelease(c.oldSema) } c.m.Unlock() } @@ -72,19 +95,19 @@ func (c *Cond) Signal() { // during the call. func (c *Cond) Broadcast() { c.m.Lock() - if c.waiters > 0 { - s := c.sema - n := c.waiters - for i := 0; i < n; i++ { - runtime.Semrelease(s) + // Wake both generations. + if c.oldWaiters > 0 { + for i := 0; i < c.oldWaiters; i++ { + runtime.Semrelease(c.oldSema) + } + c.oldWaiters = 0 + } + if c.newWaiters > 0 { + for i := 0; i < c.newWaiters; i++ { + runtime.Semrelease(c.newSema) } - // We just issued n wakeups via the semaphore s. - // To ensure that they wake up the existing waiters - // and not waiters that arrive after Broadcast returns, - // clear c.sema. The next operation will allocate - // a new one. - c.sema = nil - c.waiters = 0 + c.newWaiters = 0 + c.newSema = nil } c.m.Unlock() } diff --git a/src/pkg/sync/cond_test.go b/src/pkg/sync/cond_test.go index 846f98bf3..cefacb184 100644 --- a/src/pkg/sync/cond_test.go +++ b/src/pkg/sync/cond_test.go @@ -46,6 +46,33 @@ func TestCondSignal(t *testing.T) { c.Signal() } +func TestCondSignalGenerations(t *testing.T) { + var m Mutex + c := NewCond(&m) + n := 100 + running := make(chan bool, n) + awake := make(chan int, n) + for i := 0; i < n; i++ { + go func(i int) { + m.Lock() + running <- true + c.Wait() + awake <- i + m.Unlock() + }(i) + if i > 0 { + a := <-awake + if a != i-1 { + t.Fatalf("wrong goroutine woke up: want %d, got %d", i-1, a) + } + } + <-running + m.Lock() + c.Signal() + m.Unlock() + } +} + func TestCondBroadcast(t *testing.T) { var m Mutex c := NewCond(&m) diff --git a/src/pkg/sync/rwmutex_test.go b/src/pkg/sync/rwmutex_test.go index 9fb89f8e8..0480a6601 100644 --- a/src/pkg/sync/rwmutex_test.go +++ b/src/pkg/sync/rwmutex_test.go @@ -45,6 +45,7 @@ func doTestParallelReaders(numReaders, gomaxprocs int) { } func TestParallelReaders(t *testing.T) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(-1)) doTestParallelReaders(1, 4) doTestParallelReaders(3, 4) doTestParallelReaders(4, 2) @@ -102,6 +103,7 @@ func HammerRWMutex(gomaxprocs, numReaders, num_iterations int) { } func TestRWMutex(t *testing.T) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(-1)) n := 1000 if testing.Short() { n = 5 diff --git a/src/pkg/syscall/Makefile b/src/pkg/syscall/Makefile index 978bc94f8..9284fcc5d 100644 --- a/src/pkg/syscall/Makefile +++ b/src/pkg/syscall/Makefile @@ -17,24 +17,27 @@ GOFILES=\ ztypes_$(GOOS)_$(GOARCH).go\ GOFILES_freebsd=\ + exec_unix.go\ + route_bsd.go\ syscall_bsd.go\ syscall_unix.go\ - exec_unix.go\ GOFILES_darwin=\ + exec_unix.go\ + route_bsd.go\ syscall_bsd.go\ syscall_unix.go\ - exec_unix.go\ GOFILES_linux=\ - syscall_unix.go\ exec_unix.go\ + netlink_linux.go\ + syscall_unix.go\ GOFILES_windows=\ - exec_windows.go - + exec_windows.go\ + GOFILES_plan9=\ - exec_plan9.go + exec_plan9.go\ OFILES=\ asm_$(GOOS)_$(GOARCH).$O\ diff --git a/src/pkg/syscall/exec_windows.go b/src/pkg/syscall/exec_windows.go index 85b1c2eda..b25f4a650 100644 --- a/src/pkg/syscall/exec_windows.go +++ b/src/pkg/syscall/exec_windows.go @@ -14,7 +14,7 @@ import ( var ForkLock sync.RWMutex -// escape rewrites command line argument s as prescribed +// EscapeArg rewrites command line argument s as prescribed // in http://msdn.microsoft.com/en-us/library/ms880421. // This function returns "" (2 double quotes) if s is empty. // Alternatively, these transformations are done: @@ -23,7 +23,7 @@ var ForkLock sync.RWMutex // - every double quote (") is escaped by back slash (\); // - finally, s is wrapped with double quotes (arg -> "arg"), // but only if there is space or tab inside s. -func escape(s string) string { +func EscapeArg(s string) string { if len(s) == 0 { return "\"\"" } @@ -89,7 +89,7 @@ func makeCmdLine(args []string) string { if s != "" { s += " " } - s += escape(v) + s += EscapeArg(v) } return s } @@ -222,6 +222,7 @@ type ProcAttr struct { Env []string Files []int HideWindow bool + CmdLine string // used if non-empty, else the windows command line is built by escaping the arguments passed to StartProcess } var zeroAttributes ProcAttr @@ -252,10 +253,19 @@ func StartProcess(argv0 string, argv []string, attr *ProcAttr) (pid, handle int, } argv0p := StringToUTF16Ptr(argv0) + var cmdline string + // Windows CreateProcess takes the command line as a single string: + // use attr.CmdLine if set, else build the command line by escaping + // and joining each argument with spaces + if attr.CmdLine != "" { + cmdline = attr.CmdLine + } else { + cmdline = makeCmdLine(argv) + } + var argvp *uint16 - s := makeCmdLine(argv) - if len(s) != 0 { - argvp = StringToUTF16Ptr(s) + if len(cmdline) != 0 { + argvp = StringToUTF16Ptr(cmdline) } var dirp *uint16 diff --git a/src/pkg/syscall/mkall.sh b/src/pkg/syscall/mkall.sh index a2e6c5d71..0a4ca0326 100755 --- a/src/pkg/syscall/mkall.sh +++ b/src/pkg/syscall/mkall.sh @@ -109,12 +109,12 @@ _* | *_ | _) freebsd_386) mkerrors="$mkerrors -f -m32" mksyscall="./mksyscall.pl -l32" - mksysnum="curl -s 'http://svn.freebsd.org/viewvc/base/head/sys/kern/syscalls.master?view=markup' | ./mksysnum_freebsd.pl" + mksysnum="curl -s 'http://svn.freebsd.org/base/head/sys/kern/syscalls.master' | ./mksysnum_freebsd.pl" mktypes="godefs -gsyscall -f-m32" ;; freebsd_amd64) mkerrors="$mkerrors -f -m64" - mksysnum="curl -s 'http://svn.freebsd.org/viewvc/base/head/sys/kern/syscalls.master?view=markup' | ./mksysnum_freebsd.pl" + mksysnum="curl -s 'http://svn.freebsd.org/base/head/sys/kern/syscalls.master' | ./mksysnum_freebsd.pl" mktypes="godefs -gsyscall -f-m64" ;; darwin_386) diff --git a/src/pkg/syscall/mkerrors.sh b/src/pkg/syscall/mkerrors.sh index 0bfd9af1d..1949ebfad 100755 --- a/src/pkg/syscall/mkerrors.sh +++ b/src/pkg/syscall/mkerrors.sh @@ -21,18 +21,23 @@ includes_Linux=' #define _FILE_OFFSET_BITS 64 #define _GNU_SOURCE -#include <sys/types.h> +#include <bits/sockaddr.h> #include <sys/epoll.h> #include <sys/inotify.h> #include <sys/ioctl.h> #include <sys/mman.h> #include <sys/mount.h> #include <sys/stat.h> -#include <linux/ptrace.h> -#include <linux/wait.h> +#include <sys/types.h> +#include <linux/if_addr.h> #include <linux/if_tun.h> +#include <linux/netlink.h> #include <linux/reboot.h> +#include <linux/rtnetlink.h> +#include <linux/ptrace.h> +#include <linux/wait.h> #include <net/if.h> +#include <net/if_arp.h> #include <netpacket/packet.h> ' @@ -49,6 +54,7 @@ includes_Darwin=' #include <sys/wait.h> #include <net/bpf.h> #include <net/if.h> +#include <net/if_types.h> #include <net/route.h> #include <netinet/in.h> #include <netinet/ip.h> @@ -64,6 +70,7 @@ includes_FreeBSD=' #include <sys/wait.h> #include <net/bpf.h> #include <net/if.h> +#include <net/if_types.h> #include <net/route.h> #include <netinet/in.h> #include <netinet/ip.h> @@ -132,8 +139,10 @@ done $2 ~ /^(O|F|FD|NAME|S|PTRACE)_/ || $2 ~ /^LINUX_REBOOT_CMD_/ || $2 ~ /^LINUX_REBOOT_MAGIC[12]$/ || + $2 !~ "NLA_TYPE_MASK" && + $2 ~ /^(NETLINK|NLM|NLMSG|NLA|IFA|RTM|RTN|RTPROT|RTA|RTAX|RTNH|ARPHRD)_/ || $2 ~ /^SIOC/ || - $2 ~ /^(IFF|NET_RT|RTM|RTF|RTV|RTA|RTAX)_/ || + $2 ~ /^(IFF|IFT|NET_RT|RTM|RTF|RTV|RTA|RTAX)_/ || $2 ~ /^BIOC/ || $2 !~ /^(BPF_TIMEVAL)$/ && $2 ~ /^(BPF|DLT)_/ || diff --git a/src/pkg/syscall/mksyscall_windows.pl b/src/pkg/syscall/mksyscall_windows.pl index d92ac3d28..fb5a1272b 100755 --- a/src/pkg/syscall/mksyscall_windows.pl +++ b/src/pkg/syscall/mksyscall_windows.pl @@ -156,6 +156,7 @@ while(<>) { $text .= "\tvar _p$n uint32\n"; $text .= "\tif $name {\n\t\t_p$n = 1\n\t} else {\n\t\t_p$n = 0\n\t}\n"; push @args, "uintptr(_p$n)"; + $n++; } else { push @args, "uintptr($name)"; } diff --git a/src/pkg/syscall/netlink_linux.go b/src/pkg/syscall/netlink_linux.go new file mode 100644 index 000000000..681027ab9 --- /dev/null +++ b/src/pkg/syscall/netlink_linux.go @@ -0,0 +1,227 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Netlink sockets and messages + +package syscall + +import ( + "unsafe" +) + +// Round the length of a netlink message up to align it properly. +func nlmAlignOf(msglen int) int { + return (msglen + NLMSG_ALIGNTO - 1) & ^(NLMSG_ALIGNTO - 1) +} + +// Round the length of a netlink route attribute up to align it +// properly. +func rtaAlignOf(attrlen int) int { + return (attrlen + RTA_ALIGNTO - 1) & ^(RTA_ALIGNTO - 1) +} + +// NetlinkRouteRequest represents the request message to receive +// routing and link states from the kernel. +type NetlinkRouteRequest struct { + Header NlMsghdr + Data RtGenmsg +} + +func (rr *NetlinkRouteRequest) toWireFormat() []byte { + b := make([]byte, rr.Header.Len) + b[0] = byte(rr.Header.Len) + b[1] = byte(rr.Header.Len >> 8) + b[2] = byte(rr.Header.Len >> 16) + b[3] = byte(rr.Header.Len >> 24) + b[4] = byte(rr.Header.Type) + b[5] = byte(rr.Header.Type >> 8) + b[6] = byte(rr.Header.Flags) + b[7] = byte(rr.Header.Flags >> 8) + b[8] = byte(rr.Header.Seq) + b[9] = byte(rr.Header.Seq >> 8) + b[10] = byte(rr.Header.Seq >> 16) + b[11] = byte(rr.Header.Seq >> 24) + b[12] = byte(rr.Header.Pid) + b[13] = byte(rr.Header.Pid >> 8) + b[14] = byte(rr.Header.Pid >> 16) + b[15] = byte(rr.Header.Pid >> 24) + b[16] = byte(rr.Data.Family) + return b +} + +func newNetlinkRouteRequest(proto, seq, family int) []byte { + rr := &NetlinkRouteRequest{} + rr.Header.Len = NLMSG_HDRLEN + SizeofRtGenmsg + rr.Header.Type = uint16(proto) + rr.Header.Flags = NLM_F_DUMP | NLM_F_REQUEST + rr.Header.Seq = uint32(seq) + rr.Data.Family = uint8(family) + return rr.toWireFormat() +} + +// NetlinkRIB returns routing information base, as known as RIB, +// which consists of network facility information, states and +// parameters. +func NetlinkRIB(proto, family int) ([]byte, int) { + var ( + s int + e int + lsanl SockaddrNetlink + seq int + tab []byte + ) + + s, e = Socket(AF_NETLINK, SOCK_RAW, 0) + if e != 0 { + return nil, e + } + defer Close(s) + + lsanl.Family = AF_NETLINK + e = Bind(s, &lsanl) + if e != 0 { + return nil, e + } + + seq++ + wb := newNetlinkRouteRequest(proto, seq, family) + e = Sendto(s, wb, 0, &lsanl) + if e != 0 { + return nil, e + } + + for { + var ( + rb []byte + nr int + lsa Sockaddr + ) + + rb = make([]byte, Getpagesize()) + nr, _, e = Recvfrom(s, rb, 0) + if e != 0 { + return nil, e + } + if nr < NLMSG_HDRLEN { + return nil, EINVAL + } + rb = rb[:nr] + tab = append(tab, rb...) + + msgs, _ := ParseNetlinkMessage(rb) + for _, m := range msgs { + if lsa, e = Getsockname(s); e != 0 { + return nil, e + } + switch v := lsa.(type) { + case *SockaddrNetlink: + if m.Header.Seq != uint32(seq) || m.Header.Pid != v.Pid { + return nil, EINVAL + } + default: + return nil, EINVAL + } + if m.Header.Type == NLMSG_DONE { + goto done + } + if m.Header.Type == NLMSG_ERROR { + return nil, EINVAL + } + } + } + +done: + return tab, 0 +} + +// NetlinkMessage represents the netlink message. +type NetlinkMessage struct { + Header NlMsghdr + Data []byte +} + +// ParseNetlinkMessage parses buf as netlink messages and returns +// the slice containing the NetlinkMessage structs. +func ParseNetlinkMessage(buf []byte) ([]NetlinkMessage, int) { + var ( + h *NlMsghdr + dbuf []byte + dlen int + e int + msgs []NetlinkMessage + ) + + for len(buf) >= NLMSG_HDRLEN { + h, dbuf, dlen, e = netlinkMessageHeaderAndData(buf) + if e != 0 { + break + } + m := NetlinkMessage{} + m.Header = *h + m.Data = dbuf[:h.Len-NLMSG_HDRLEN] + msgs = append(msgs, m) + buf = buf[dlen:] + } + + return msgs, e +} + +func netlinkMessageHeaderAndData(buf []byte) (*NlMsghdr, []byte, int, int) { + h := (*NlMsghdr)(unsafe.Pointer(&buf[0])) + if h.Len < NLMSG_HDRLEN || int(h.Len) > len(buf) { + return nil, nil, 0, EINVAL + } + return h, buf[NLMSG_HDRLEN:], nlmAlignOf(int(h.Len)), 0 +} + +// NetlinkRouteAttr represents the netlink route attribute. +type NetlinkRouteAttr struct { + Attr RtAttr + Value []byte +} + +// ParseNetlinkRouteAttr parses msg's payload as netlink route +// attributes and returns the slice containing the NetlinkRouteAttr +// structs. +func ParseNetlinkRouteAttr(msg *NetlinkMessage) ([]NetlinkRouteAttr, int) { + var ( + buf []byte + a *RtAttr + alen int + vbuf []byte + e int + attrs []NetlinkRouteAttr + ) + + switch msg.Header.Type { + case RTM_NEWLINK: + buf = msg.Data[SizeofIfInfomsg:] + case RTM_NEWADDR: + buf = msg.Data[SizeofIfAddrmsg:] + default: + return nil, EINVAL + } + + for len(buf) >= SizeofRtAttr { + a, vbuf, alen, e = netlinkRouteAttrAndValue(buf) + if e != 0 { + break + } + ra := NetlinkRouteAttr{} + ra.Attr = *a + ra.Value = vbuf[:a.Len-SizeofRtAttr] + attrs = append(attrs, ra) + buf = buf[alen:] + } + + return attrs, 0 +} + +func netlinkRouteAttrAndValue(buf []byte) (*RtAttr, []byte, int, int) { + h := (*RtAttr)(unsafe.Pointer(&buf[0])) + if h.Len < SizeofRtAttr || int(h.Len) > len(buf) { + return nil, nil, 0, EINVAL + } + return h, buf[SizeofRtAttr:], rtaAlignOf(int(h.Len)), 0 +} diff --git a/src/pkg/syscall/route_bsd.go b/src/pkg/syscall/route_bsd.go new file mode 100644 index 000000000..79a8793d5 --- /dev/null +++ b/src/pkg/syscall/route_bsd.go @@ -0,0 +1,180 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Routing sockets and messages + +package syscall + +import ( + "unsafe" +) + +const darwinAMD64 = OS == "darwin" && ARCH == "amd64" + +// Round the length of a raw sockaddr up to align it properly. +func rsaAlignOf(salen int) int { + salign := sizeofPtr + // NOTE: It seems like 64-bit Darwin kernel still requires 32-bit + // aligned access to BSD subsystem. + if darwinAMD64 { + salign = 4 + } + if salen == 0 { + return salign + } + return (salen + salign - 1) & ^(salign - 1) +} + +// RouteRIB returns routing information base, as known as RIB, +// which consists of network facility information, states and +// parameters. +func RouteRIB(facility, param int) ([]byte, int) { + var ( + tab []byte + e int + ) + + mib := []_C_int{CTL_NET, AF_ROUTE, 0, 0, _C_int(facility), _C_int(param)} + + // Find size. + n := uintptr(0) + if e = sysctl(mib, nil, &n, nil, 0); e != 0 { + return nil, e + } + if n == 0 { + return nil, 0 + } + + tab = make([]byte, n) + if e = sysctl(mib, &tab[0], &n, nil, 0); e != 0 { + return nil, e + } + + return tab[:n], 0 +} + +// RoutingMessage represents a routing message. +type RoutingMessage interface { + sockaddr() []Sockaddr +} + +const anyMessageLen = unsafe.Sizeof(anyMessage{}) + +type anyMessage struct { + Msglen uint16 + Version uint8 + Type uint8 +} + +func (any *anyMessage) toRoutingMessage(buf []byte) RoutingMessage { + switch any.Type { + case RTM_ADD, RTM_DELETE, RTM_CHANGE, RTM_GET, RTM_LOSING, RTM_REDIRECT, RTM_MISS, RTM_LOCK, RTM_RESOLVE: + p := (*RouteMessage)(unsafe.Pointer(any)) + rtm := &RouteMessage{} + rtm.Header = p.Header + rtm.Data = buf[SizeofRtMsghdr:any.Msglen] + return rtm + case RTM_IFINFO: + p := (*InterfaceMessage)(unsafe.Pointer(any)) + ifm := &InterfaceMessage{} + ifm.Header = p.Header + ifm.Data = buf[SizeofIfMsghdr:any.Msglen] + return ifm + case RTM_NEWADDR, RTM_DELADDR: + p := (*InterfaceAddrMessage)(unsafe.Pointer(any)) + ifam := &InterfaceAddrMessage{} + ifam.Header = p.Header + ifam.Data = buf[SizeofIfaMsghdr:any.Msglen] + return ifam + case RTM_NEWMADDR, RTM_DELMADDR: + // TODO: implement this in the near future + } + return nil +} + +// RouteMessage represents a routing message containing routing +// entries. +type RouteMessage struct { + Header RtMsghdr + Data []byte +} + +func (m *RouteMessage) sockaddr() (sas []Sockaddr) { + // TODO: implement this in the near future + return nil +} + +// InterfaceMessage represents a routing message containing +// network interface entries. +type InterfaceMessage struct { + Header IfMsghdr + Data []byte +} + +func (m *InterfaceMessage) sockaddr() (sas []Sockaddr) { + if m.Header.Addrs&RTA_IFP == 0 { + return nil + } + sa, e := anyToSockaddr((*RawSockaddrAny)(unsafe.Pointer(&m.Data[0]))) + if e != 0 { + return nil + } + return append(sas, sa) +} + +// InterfaceAddrMessage represents a routing message containing +// network interface address entries. +type InterfaceAddrMessage struct { + Header IfaMsghdr + Data []byte +} + +const rtaMask = RTA_IFA | RTA_NETMASK | RTA_BRD + +func (m *InterfaceAddrMessage) sockaddr() (sas []Sockaddr) { + if m.Header.Addrs&rtaMask == 0 { + return nil + } + + buf := m.Data[:] + for i := uint(0); i < RTAX_MAX; i++ { + if m.Header.Addrs&rtaMask&(1<<i) == 0 { + continue + } + rsa := (*RawSockaddr)(unsafe.Pointer(&buf[0])) + switch i { + case RTAX_IFA: + sa, e := anyToSockaddr((*RawSockaddrAny)(unsafe.Pointer(rsa))) + if e != 0 { + return nil + } + sas = append(sas, sa) + case RTAX_NETMASK, RTAX_BRD: + // nothing to do + } + buf = buf[rsaAlignOf(int(rsa.Len)):] + } + + return sas +} + +// ParseRoutingMessage parses buf as routing messages and returns +// the slice containing the RoutingMessage interfaces. +func ParseRoutingMessage(buf []byte) (msgs []RoutingMessage, errno int) { + for len(buf) >= anyMessageLen { + any := (*anyMessage)(unsafe.Pointer(&buf[0])) + if any.Version != RTM_VERSION { + return nil, EINVAL + } + msgs = append(msgs, any.toRoutingMessage(buf)) + buf = buf[any.Msglen:] + } + return msgs, 0 +} + +// ParseRoutingMessage parses msg's payload as raw sockaddrs and +// returns the slice containing the Sockaddr interfaces. +func ParseRoutingSockaddr(msg RoutingMessage) (sas []Sockaddr, errno int) { + return append(sas, msg.sockaddr()...), 0 +} diff --git a/src/pkg/syscall/syscall_bsd.go b/src/pkg/syscall/syscall_bsd.go index 9f1244f13..89bcc7f0e 100644 --- a/src/pkg/syscall/syscall_bsd.go +++ b/src/pkg/syscall/syscall_bsd.go @@ -22,7 +22,6 @@ const ImplementsGetwd = false func Getwd() (string, int) { return "", ENOTSUP } - /* * Wrapped */ @@ -191,9 +190,10 @@ func (sa *SockaddrInet4) sockaddr() (uintptr, _Socklen, int) { } type SockaddrInet6 struct { - Port int - Addr [16]byte - raw RawSockaddrInet6 + Port int + ZoneId uint32 + Addr [16]byte + raw RawSockaddrInet6 } func (sa *SockaddrInet6) sockaddr() (uintptr, _Socklen, int) { @@ -205,6 +205,7 @@ func (sa *SockaddrInet6) sockaddr() (uintptr, _Socklen, int) { p := (*[2]byte)(unsafe.Pointer(&sa.raw.Port)) p[0] = byte(sa.Port >> 8) p[1] = byte(sa.Port) + sa.raw.Scope_id = sa.ZoneId for i := 0; i < len(sa.Addr); i++ { sa.raw.Addr[i] = sa.Addr[i] } @@ -297,6 +298,7 @@ func anyToSockaddr(rsa *RawSockaddrAny) (Sockaddr, int) { sa := new(SockaddrInet6) p := (*[2]byte)(unsafe.Pointer(&pp.Port)) sa.Port = int(p[0])<<8 + int(p[1]) + sa.ZoneId = pp.Scope_id for i := 0; i < len(sa.Addr); i++ { sa.Addr[i] = pp.Addr[i] } @@ -389,7 +391,11 @@ func SetsockoptLinger(fd, level, opt int, l *Linger) (errno int) { return setsockopt(fd, level, opt, uintptr(unsafe.Pointer(l)), unsafe.Sizeof(*l)) } -func SetsockoptIpMreq(fd, level, opt int, mreq *IpMreq) (errno int) { +func SetsockoptIPMreq(fd, level, opt int, mreq *IPMreq) (errno int) { + return setsockopt(fd, level, opt, uintptr(unsafe.Pointer(mreq)), unsafe.Sizeof(*mreq)) +} + +func SetsockoptIPv6Mreq(fd, level, opt int, mreq *IPv6Mreq) (errno int) { return setsockopt(fd, level, opt, uintptr(unsafe.Pointer(mreq)), unsafe.Sizeof(*mreq)) } @@ -516,27 +522,6 @@ func SysctlUint32(name string) (value uint32, errno int) { return *(*uint32)(unsafe.Pointer(&buf[0])), 0 } -func SysctlNetRoute(fourth, fifth, sixth int) (value []byte, errno int) { - mib := []_C_int{CTL_NET, AF_ROUTE, 0, _C_int(fourth), _C_int(fifth), _C_int(sixth)} - - // Find size. - n := uintptr(0) - if errno = sysctl(mib, nil, &n, nil, 0); errno != 0 { - return nil, errno - } - if n == 0 { - return nil, 0 - } - - // Read into buffer of that size. - b := make([]byte, n) - if errno = sysctl(mib, &b[0], &n, nil, 0); errno != 0 { - return nil, errno - } - - return b[0:n], 0 -} - //sys utimes(path string, timeval *[2]Timeval) (errno int) func Utimes(path string, tv []Timeval) (errno int) { if len(tv) != 2 { diff --git a/src/pkg/syscall/syscall_darwin.go b/src/pkg/syscall/syscall_darwin.go index 30b57cf55..9e153b73d 100644 --- a/src/pkg/syscall/syscall_darwin.go +++ b/src/pkg/syscall/syscall_darwin.go @@ -56,6 +56,11 @@ func ParseDirent(buf []byte, max int, names []string) (consumed int, count int, return origlen - len(buf), count, names } +// TODO +func Sendfile(outfd int, infd int, offset *int64, count int) (written int, errno int) { + return -1, ENOSYS +} + /* * Wrapped */ diff --git a/src/pkg/syscall/syscall_freebsd.go b/src/pkg/syscall/syscall_freebsd.go index 242503dd7..a38c8ecd3 100644 --- a/src/pkg/syscall/syscall_freebsd.go +++ b/src/pkg/syscall/syscall_freebsd.go @@ -56,6 +56,11 @@ func ParseDirent(buf []byte, max int, names []string) (consumed int, count int, return origlen - len(buf), count, names } +// TODO +func Sendfile(outfd int, infd int, offset *int64, count int) (written int, errno int) { + return -1, ENOSYS +} + /* * Exposed directly */ diff --git a/src/pkg/syscall/syscall_linux.go b/src/pkg/syscall/syscall_linux.go index 4a3797c20..63682d23c 100644 --- a/src/pkg/syscall/syscall_linux.go +++ b/src/pkg/syscall/syscall_linux.go @@ -219,9 +219,10 @@ func (sa *SockaddrInet4) sockaddr() (uintptr, _Socklen, int) { } type SockaddrInet6 struct { - Port int - Addr [16]byte - raw RawSockaddrInet6 + Port int + ZoneId uint32 + Addr [16]byte + raw RawSockaddrInet6 } func (sa *SockaddrInet6) sockaddr() (uintptr, _Socklen, int) { @@ -232,6 +233,7 @@ func (sa *SockaddrInet6) sockaddr() (uintptr, _Socklen, int) { p := (*[2]byte)(unsafe.Pointer(&sa.raw.Port)) p[0] = byte(sa.Port >> 8) p[1] = byte(sa.Port) + sa.raw.Scope_id = sa.ZoneId for i := 0; i < len(sa.Addr); i++ { sa.raw.Addr[i] = sa.Addr[i] } @@ -290,8 +292,33 @@ func (sa *SockaddrLinklayer) sockaddr() (uintptr, _Socklen, int) { return uintptr(unsafe.Pointer(&sa.raw)), SizeofSockaddrLinklayer, 0 } +type SockaddrNetlink struct { + Family uint16 + Pad uint16 + Pid uint32 + Groups uint32 + raw RawSockaddrNetlink +} + +func (sa *SockaddrNetlink) sockaddr() (uintptr, _Socklen, int) { + sa.raw.Family = AF_NETLINK + sa.raw.Pad = sa.Pad + sa.raw.Pid = sa.Pid + sa.raw.Groups = sa.Groups + return uintptr(unsafe.Pointer(&sa.raw)), SizeofSockaddrNetlink, 0 +} + func anyToSockaddr(rsa *RawSockaddrAny) (Sockaddr, int) { switch rsa.Addr.Family { + case AF_NETLINK: + pp := (*RawSockaddrNetlink)(unsafe.Pointer(rsa)) + sa := new(SockaddrNetlink) + sa.Family = pp.Family + sa.Pad = pp.Pad + sa.Pid = pp.Pid + sa.Groups = pp.Groups + return sa, 0 + case AF_PACKET: pp := (*RawSockaddrLinklayer)(unsafe.Pointer(rsa)) sa := new(SockaddrLinklayer) @@ -345,6 +372,7 @@ func anyToSockaddr(rsa *RawSockaddrAny) (Sockaddr, int) { sa := new(SockaddrInet6) p := (*[2]byte)(unsafe.Pointer(&pp.Port)) sa.Port = int(p[0])<<8 + int(p[1]) + sa.ZoneId = pp.Scope_id for i := 0; i < len(sa.Addr); i++ { sa.Addr[i] = pp.Addr[i] } @@ -435,7 +463,11 @@ func SetsockoptLinger(fd, level, opt int, l *Linger) (errno int) { return setsockopt(fd, level, opt, uintptr(unsafe.Pointer(l)), unsafe.Sizeof(*l)) } -func SetsockoptIpMreq(fd, level, opt int, mreq *IpMreq) (errno int) { +func SetsockoptIPMreq(fd, level, opt int, mreq *IPMreq) (errno int) { + return setsockopt(fd, level, opt, uintptr(unsafe.Pointer(mreq)), unsafe.Sizeof(*mreq)) +} + +func SetsockoptIPv6Mreq(fd, level, opt int, mreq *IPv6Mreq) (errno int) { return setsockopt(fd, level, opt, uintptr(unsafe.Pointer(mreq)), unsafe.Sizeof(*mreq)) } @@ -489,7 +521,7 @@ func Recvmsg(fd int, p, oob []byte, flags int) (n, oobn int, recvflags int, from oobn = int(msg.Controllen) recvflags = int(msg.Flags) // source address is only specified if the socket is unconnected - if rsa.Addr.Family != 0 { + if rsa.Addr.Family != AF_UNSPEC { from, errno = anyToSockaddr(&rsa) } return @@ -932,7 +964,6 @@ func Munmap(b []byte) (errno int) { // Semget // Semop // Semtimedop -// Sendfile // SetMempolicy // SetRobustList // SetThreadArea diff --git a/src/pkg/syscall/syscall_linux_386.go b/src/pkg/syscall/syscall_linux_386.go index 2b6bdebf8..5195179a2 100644 --- a/src/pkg/syscall/syscall_linux_386.go +++ b/src/pkg/syscall/syscall_linux_386.go @@ -41,6 +41,7 @@ func NsecToTimeval(nsec int64) (tv Timeval) { //sys Lstat(path string, stat *Stat_t) (errno int) = SYS_LSTAT64 //sys Pread(fd int, p []byte, offset int64) (n int, errno int) = SYS_PREAD64 //sys Pwrite(fd int, p []byte, offset int64) (n int, errno int) = SYS_PWRITE64 +//sys Sendfile(outfd int, infd int, offset *int64, count int) (written int, errno int) = SYS_SENDFILE64 //sys Setfsgid(gid int) (errno int) = SYS_SETFSGID32 //sys Setfsuid(uid int) (errno int) = SYS_SETFSUID32 //sysnb Setgid(gid int) (errno int) = SYS_SETGID32 diff --git a/src/pkg/syscall/syscall_linux_amd64.go b/src/pkg/syscall/syscall_linux_amd64.go index f2a4acfe9..db9524668 100644 --- a/src/pkg/syscall/syscall_linux_amd64.go +++ b/src/pkg/syscall/syscall_linux_amd64.go @@ -22,6 +22,7 @@ package syscall //sys Pwrite(fd int, p []byte, offset int64) (n int, errno int) = SYS_PWRITE64 //sys Seek(fd int, offset int64, whence int) (off int64, errno int) = SYS_LSEEK //sys Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, errno int) +//sys Sendfile(outfd int, infd int, offset *int64, count int) (written int, errno int) //sys Setfsgid(gid int) (errno int) //sys Setfsuid(uid int) (errno int) //sysnb Setgid(gid int) (errno int) diff --git a/src/pkg/syscall/syscall_linux_arm.go b/src/pkg/syscall/syscall_linux_arm.go index 458745885..37845301f 100644 --- a/src/pkg/syscall/syscall_linux_arm.go +++ b/src/pkg/syscall/syscall_linux_arm.go @@ -92,6 +92,7 @@ func Seek(fd int, offset int64, whence int) (newoffset int64, errno int) //sys Lchown(path string, uid int, gid int) (errno int) //sys Listen(s int, n int) (errno int) //sys Lstat(path string, stat *Stat_t) (errno int) = SYS_LSTAT64 +//sys Sendfile(outfd int, infd int, offset *int64, count int) (written int, errno int) = SYS_SENDFILE64 //sys Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, errno int) = SYS__NEWSELECT //sys Setfsgid(gid int) (errno int) //sys Setfsuid(uid int) (errno int) diff --git a/src/pkg/syscall/syscall_plan9.go b/src/pkg/syscall/syscall_plan9.go index 831cbddb2..730126f23 100644 --- a/src/pkg/syscall/syscall_plan9.go +++ b/src/pkg/syscall/syscall_plan9.go @@ -327,6 +327,11 @@ func Getgroups() (gids []int, err Error) { return make([]int, 0), nil } +// TODO +func Sendfile(outfd int, infd int, offset *int64, count int) (written int, errno int) { + return -1, ENOSYS +} + //sys Dup(oldfd int, newfd int) (fd int, err Error) //sys Open(path string, mode int) (fd int, err Error) //sys Create(path string, mode int, perm uint32) (fd int, err Error) diff --git a/src/pkg/syscall/syscall_windows.go b/src/pkg/syscall/syscall_windows.go index 1fbb3ccbf..d01664d12 100644 --- a/src/pkg/syscall/syscall_windows.go +++ b/src/pkg/syscall/syscall_windows.go @@ -102,6 +102,11 @@ func getSysProcAddr(m uint32, pname string) uintptr { // Implemented in ../runtime/windows/syscall.cgo func NewCallback(fn interface{}) uintptr +// TODO +func Sendfile(outfd int, infd int, offset *int64, count int) (written int, errno int) { + return -1, ENOSYS +} + // windows api calls //sys GetLastError() (lasterrno int) @@ -136,8 +141,9 @@ func NewCallback(fn interface{}) uintptr //sys GetQueuedCompletionStatus(cphandle int32, qty *uint32, key *uint32, overlapped **Overlapped, timeout uint32) (errno int) //sys CancelIo(s uint32) (errno int) //sys CreateProcess(appName *uint16, commandLine *uint16, procSecurity *SecurityAttributes, threadSecurity *SecurityAttributes, inheritHandles bool, creationFlags uint32, env *uint16, currentDir *uint16, startupInfo *StartupInfo, outProcInfo *ProcessInformation) (errno int) = CreateProcessW -//sys OpenProcess(da uint32, inheritHandle bool, pid uint32) (handle uint32, errno int) -//sys GetExitCodeProcess(handle uint32, exitcode *uint32) (errno int) +//sys OpenProcess(da uint32, inheritHandle bool, pid uint32) (handle int32, errno int) +//sys TerminateProcess(handle int32, exitcode uint32) (errno int) +//sys GetExitCodeProcess(handle int32, exitcode *uint32) (errno int) //sys GetStartupInfo(startupInfo *StartupInfo) (errno int) = GetStartupInfoW //sys GetCurrentProcess() (pseudoHandle int32, errno int) //sys DuplicateHandle(hSourceProcessHandle int32, hSourceHandle int32, hTargetProcessHandle int32, lpTargetHandle *int32, dwDesiredAccess uint32, bInheritHandle bool, dwOptions uint32) (errno int) @@ -161,6 +167,12 @@ func NewCallback(fn interface{}) uintptr //sys SetHandleInformation(handle int32, mask uint32, flags uint32) (errno int) //sys FlushFileBuffers(handle int32) (errno int) //sys GetFullPathName(path *uint16, buflen uint32, buf *uint16, fname **uint16) (n uint32, errno int) = kernel32.GetFullPathNameW +//sys CreateFileMapping(fhandle int32, sa *SecurityAttributes, prot uint32, maxSizeHigh uint32, maxSizeLow uint32, name *uint16) (handle int32, errno int) = kernel32.CreateFileMappingW +//sys MapViewOfFile(handle int32, access uint32, offsetHigh uint32, offsetLow uint32, length uintptr) (addr uintptr, errno int) +//sys UnmapViewOfFile(addr uintptr) (errno int) +//sys FlushViewOfFile(addr uintptr, length uintptr) (errno int) +//sys VirtualLock(addr uintptr, length uintptr) (errno int) +//sys VirtualUnlock(addr uintptr, length uintptr) (errno int) // syscall interface implementation for other packages @@ -219,16 +231,13 @@ func Open(path string, mode int, perm uint32) (fd int, errno int) { } var createmode uint32 switch { - case mode&O_CREAT != 0: - switch { - case mode&O_EXCL != 0: - createmode = CREATE_NEW - case mode&O_APPEND != 0: - createmode = OPEN_ALWAYS - default: - createmode = CREATE_ALWAYS - } - case mode&O_TRUNC != 0: + case mode&(O_CREAT|O_EXCL) == (O_CREAT | O_EXCL): + createmode = CREATE_NEW + case mode&(O_CREAT|O_TRUNC) == (O_CREAT | O_TRUNC): + createmode = CREATE_ALWAYS + case mode&O_CREAT == O_CREAT: + createmode = OPEN_ALWAYS + case mode&O_TRUNC == O_TRUNC: createmode = TRUNCATE_EXISTING default: createmode = OPEN_EXISTING @@ -525,8 +534,9 @@ func (sa *SockaddrInet4) sockaddr() (uintptr, int32, int) { } type SockaddrInet6 struct { - Port int - Addr [16]byte + Port int + ZoneId uint32 + Addr [16]byte } func (sa *SockaddrInet6) sockaddr() (uintptr, int32, int) { @@ -671,20 +681,22 @@ const ( IP_DROP_MEMBERSHIP ) -type IpMreq struct { +type IPMreq struct { Multiaddr [4]byte /* in_addr */ Interface [4]byte /* in_addr */ } -func SetsockoptLinger(fd, level, opt int, l *Linger) (errno int) { return EWINDOWS } -func SetsockoptIpMreq(fd, level, opt int, mreq *IpMreq) (errno int) { return EWINDOWS } -func BindToDevice(fd int, device string) (errno int) { return EWINDOWS } +type IPv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} -// TODO(brainman): fix all needed for os +func SetsockoptLinger(fd, level, opt int, l *Linger) (errno int) { return EWINDOWS } +func SetsockoptIPMreq(fd, level, opt int, mreq *IPMreq) (errno int) { return EWINDOWS } +func SetsockoptIPv6Mreq(fd, level, opt int, mreq *IPv6Mreq) (errno int) { return EWINDOWS } +func BindToDevice(fd int, device string) (errno int) { return EWINDOWS } -const ( - SIGTRAP = 5 -) +// TODO(brainman): fix all needed for os func Getpid() (pid int) { return -1 } func Getppid() (ppid int) { return -1 } diff --git a/src/pkg/syscall/types_darwin.c b/src/pkg/syscall/types_darwin.c index 666923a68..ecccd5bd9 100644 --- a/src/pkg/syscall/types_darwin.c +++ b/src/pkg/syscall/types_darwin.c @@ -39,8 +39,7 @@ Input to godefs. See also mkerrors.sh and mkall.sh // Machine characteristics; for internal use. -enum -{ +enum { $sizeofPtr = sizeof(void*), $sizeofShort = sizeof(short), $sizeofInt = sizeof(int), @@ -48,7 +47,6 @@ enum $sizeofLongLong = sizeof(long long), }; - // Basic types typedef short $_C_short; @@ -71,8 +69,7 @@ typedef gid_t $_Gid_t; // Files -enum -{ +enum { $O_CLOEXEC = 0, // not supported }; @@ -86,8 +83,6 @@ typedef struct log2phys $Log2phys_t; typedef struct dirent $Dirent; -// Wait status. - // Sockets union sockaddr_all { @@ -112,9 +107,11 @@ typedef struct sockaddr_any $RawSockaddrAny; typedef socklen_t $_Socklen; typedef struct linger $Linger; typedef struct iovec $Iovec; -typedef struct ip_mreq $IpMreq; +typedef struct ip_mreq $IPMreq; +typedef struct ipv6_mreq $IPv6Mreq; typedef struct msghdr $Msghdr; typedef struct cmsghdr $Cmsghdr; +typedef struct in6_pktinfo $Inet6Pktinfo; enum { $SizeofSockaddrInet4 = sizeof(struct sockaddr_in), @@ -123,19 +120,21 @@ enum { $SizeofSockaddrUnix = sizeof(struct sockaddr_un), $SizeofSockaddrDatalink = sizeof(struct sockaddr_dl), $SizeofLinger = sizeof(struct linger), - $SizeofIpMreq = sizeof(struct ip_mreq), + $SizeofIPMreq = sizeof(struct ip_mreq), + $SizeofIPv6Mreq = sizeof(struct ipv6_mreq), $SizeofMsghdr = sizeof(struct msghdr), $SizeofCmsghdr = sizeof(struct cmsghdr), + $SizeofInet6Pktinfo = sizeof(struct in6_pktinfo), }; // Ptrace requests + enum { $PTRACE_TRACEME = PT_TRACE_ME, $PTRACE_CONT = PT_CONTINUE, $PTRACE_KILL = PT_KILL, }; - // Events (kqueue, kevent) typedef struct kevent $Kevent_t; diff --git a/src/pkg/syscall/types_freebsd.c b/src/pkg/syscall/types_freebsd.c index 9d65683ef..97636550a 100644 --- a/src/pkg/syscall/types_freebsd.c +++ b/src/pkg/syscall/types_freebsd.c @@ -35,8 +35,7 @@ Input to godefs. See also mkerrors.sh and mkall.sh // Machine characteristics; for internal use. -enum -{ +enum { $sizeofPtr = sizeof(void*), $sizeofShort = sizeof(short), $sizeofInt = sizeof(int), @@ -44,7 +43,6 @@ enum $sizeofLongLong = sizeof(long long), }; - // Basic types typedef short $_C_short; @@ -66,13 +64,11 @@ typedef gid_t $_Gid_t; // Files -enum -{ +enum { $O_CLOEXEC = 0, // not supported }; -enum -{ // Directory mode bits +enum { // Directory mode bits $S_IFMT = S_IFMT, $S_IFIFO = S_IFIFO, $S_IFCHR = S_IFCHR, @@ -95,8 +91,6 @@ typedef struct flock $Flock_t; typedef struct dirent $Dirent; -// Wait status. - // Sockets union sockaddr_all { @@ -121,9 +115,11 @@ typedef struct sockaddr_any $RawSockaddrAny; typedef socklen_t $_Socklen; typedef struct linger $Linger; typedef struct iovec $Iovec; -typedef struct ip_mreq $IpMreq; +typedef struct ip_mreq $IPMreq; +typedef struct ipv6_mreq $IPv6Mreq; typedef struct msghdr $Msghdr; typedef struct cmsghdr $Cmsghdr; +typedef struct in6_pktinfo $Inet6Pktinfo; enum { $SizeofSockaddrInet4 = sizeof(struct sockaddr_in), @@ -132,19 +128,21 @@ enum { $SizeofSockaddrUnix = sizeof(struct sockaddr_un), $SizeofSockaddrDatalink = sizeof(struct sockaddr_dl), $SizeofLinger = sizeof(struct linger), - $SizeofIpMreq = sizeof(struct ip_mreq), + $SizeofIPMreq = sizeof(struct ip_mreq), + $SizeofIPv6Mreq = sizeof(struct ipv6_mreq), $SizeofMsghdr = sizeof(struct msghdr), $SizeofCmsghdr = sizeof(struct cmsghdr), + $SizeofInet6Pktinfo = sizeof(struct in6_pktinfo), }; // Ptrace requests + enum { $PTRACE_TRACEME = PT_TRACE_ME, $PTRACE_CONT = PT_CONTINUE, $PTRACE_KILL = PT_KILL, }; - // Events (kqueue, kevent) typedef struct kevent $Kevent_t; diff --git a/src/pkg/syscall/types_linux.c b/src/pkg/syscall/types_linux.c index ec94c84df..ce7f96764 100644 --- a/src/pkg/syscall/types_linux.c +++ b/src/pkg/syscall/types_linux.c @@ -38,6 +38,8 @@ Input to godefs. See also mkerrors.sh and mkall.sh #include <sys/user.h> #include <sys/utsname.h> #include <sys/wait.h> +#include <linux/netlink.h> +#include <linux/rtnetlink.h> #include <time.h> #include <unistd.h> #include <ustat.h> @@ -45,8 +47,7 @@ Input to godefs. See also mkerrors.sh and mkall.sh // Machine characteristics; for internal use. -enum -{ +enum { $sizeofPtr = sizeof(void*), $sizeofShort = sizeof(short), $sizeofInt = sizeof(int), @@ -93,6 +94,7 @@ union sockaddr_all { struct sockaddr_in6 s3; struct sockaddr_un s4; struct sockaddr_ll s5; + struct sockaddr_nl s6; }; struct sockaddr_any { @@ -104,14 +106,17 @@ typedef struct sockaddr_in $RawSockaddrInet4; typedef struct sockaddr_in6 $RawSockaddrInet6; typedef struct sockaddr_un $RawSockaddrUnix; typedef struct sockaddr_ll $RawSockaddrLinklayer; +typedef struct sockaddr_nl $RawSockaddrNetlink; typedef struct sockaddr $RawSockaddr; typedef struct sockaddr_any $RawSockaddrAny; typedef socklen_t $_Socklen; typedef struct linger $Linger; typedef struct iovec $Iovec; -typedef struct ip_mreq $IpMreq; +typedef struct ip_mreq $IPMreq; +typedef struct ipv6_mreq $IPv6Mreq; typedef struct msghdr $Msghdr; typedef struct cmsghdr $Cmsghdr; +typedef struct in6_pktinfo $Inet6Pktinfo; typedef struct ucred $Ucred; enum { @@ -120,22 +125,114 @@ enum { $SizeofSockaddrAny = sizeof(struct sockaddr_any), $SizeofSockaddrUnix = sizeof(struct sockaddr_un), $SizeofSockaddrLinklayer = sizeof(struct sockaddr_ll), + $SizeofSockaddrNetlink = sizeof(struct sockaddr_nl), $SizeofLinger = sizeof(struct linger), - $SizeofIpMreq = sizeof(struct ip_mreq), + $SizeofIPMreq = sizeof(struct ip_mreq), + $SizeofIPv6Mreq = sizeof(struct ipv6_mreq), $SizeofMsghdr = sizeof(struct msghdr), $SizeofCmsghdr = sizeof(struct cmsghdr), + $SizeofInet6Pktinfo = sizeof(struct in6_pktinfo), $SizeofUcred = sizeof(struct ucred), }; +// Netlink routing and interface messages + +enum { + $IFA_UNSPEC = IFA_UNSPEC, + $IFA_ADDRESS = IFA_ADDRESS, + $IFA_LOCAL = IFA_LOCAL, + $IFA_LABEL = IFA_LABEL, + $IFA_BROADCAST = IFA_BROADCAST, + $IFA_ANYCAST = IFA_ANYCAST, + $IFA_CACHEINFO = IFA_CACHEINFO, + $IFA_MULTICAST = IFA_MULTICAST, + $IFLA_UNSPEC = IFLA_UNSPEC, + $IFLA_ADDRESS = IFLA_ADDRESS, + $IFLA_BROADCAST = IFLA_BROADCAST, + $IFLA_IFNAME = IFLA_IFNAME, + $IFLA_MTU = IFLA_MTU, + $IFLA_LINK = IFLA_LINK, + $IFLA_QDISC = IFLA_QDISC, + $IFLA_STATS = IFLA_STATS, + $IFLA_COST = IFLA_COST, + $IFLA_PRIORITY = IFLA_PRIORITY, + $IFLA_MASTER = IFLA_MASTER, + $IFLA_WIRELESS = IFLA_WIRELESS, + $IFLA_PROTINFO = IFLA_PROTINFO, + $IFLA_TXQLEN = IFLA_TXQLEN, + $IFLA_MAP = IFLA_MAP, + $IFLA_WEIGHT = IFLA_WEIGHT, + $IFLA_OPERSTATE = IFLA_OPERSTATE, + $IFLA_LINKMODE = IFLA_LINKMODE, + $IFLA_LINKINFO = IFLA_LINKINFO, + $IFLA_NET_NS_PID = IFLA_NET_NS_PID, + $IFLA_IFALIAS = IFLA_IFALIAS, + $IFLA_MAX = IFLA_MAX, + $RT_SCOPE_UNIVERSE = RT_SCOPE_UNIVERSE, + $RT_SCOPE_SITE = RT_SCOPE_SITE, + $RT_SCOPE_LINK = RT_SCOPE_LINK, + $RT_SCOPE_HOST = RT_SCOPE_HOST, + $RT_SCOPE_NOWHERE = RT_SCOPE_NOWHERE, + $RT_TABLE_UNSPEC = RT_TABLE_UNSPEC, + $RT_TABLE_COMPAT = RT_TABLE_COMPAT, + $RT_TABLE_DEFAULT = RT_TABLE_DEFAULT, + $RT_TABLE_MAIN = RT_TABLE_MAIN, + $RT_TABLE_LOCAL = RT_TABLE_LOCAL, + $RT_TABLE_MAX = RT_TABLE_MAX, + $RTA_UNSPEC = RTA_UNSPEC, + $RTA_DST = RTA_DST, + $RTA_SRC = RTA_SRC, + $RTA_IIF = RTA_IIF, + $RTA_OIF = RTA_OIF, + $RTA_GATEWAY = RTA_GATEWAY, + $RTA_PRIORITY = RTA_PRIORITY, + $RTA_PREFSRC = RTA_PREFSRC, + $RTA_METRICS = RTA_METRICS, + $RTA_MULTIPATH = RTA_MULTIPATH, + $RTA_FLOW = RTA_FLOW, + $RTA_CACHEINFO = RTA_CACHEINFO, + $RTA_TABLE = RTA_TABLE, + $RTN_UNSPEC = RTN_UNSPEC, + $RTN_UNICAST = RTN_UNICAST, + $RTN_LOCAL = RTN_LOCAL, + $RTN_BROADCAST = RTN_BROADCAST, + $RTN_ANYCAST = RTN_ANYCAST, + $RTN_MULTICAST = RTN_MULTICAST, + $RTN_BLACKHOLE = RTN_BLACKHOLE, + $RTN_UNREACHABLE = RTN_UNREACHABLE, + $RTN_PROHIBIT = RTN_PROHIBIT, + $RTN_THROW = RTN_THROW, + $RTN_NAT = RTN_NAT, + $RTN_XRESOLVE = RTN_XRESOLVE, + $SizeofNlMsghdr = sizeof(struct nlmsghdr), + $SizeofNlMsgerr = sizeof(struct nlmsgerr), + $SizeofRtGenmsg = sizeof(struct rtgenmsg), + $SizeofNlAttr = sizeof(struct nlattr), + $SizeofRtAttr = sizeof(struct rtattr), + $SizeofIfInfomsg = sizeof(struct ifinfomsg), + $SizeofIfAddrmsg = sizeof(struct ifaddrmsg), + $SizeofRtmsg = sizeof(struct rtmsg), + $SizeofRtNexthop = sizeof(struct rtnexthop), +}; + +typedef struct nlmsghdr $NlMsghdr; +typedef struct nlmsgerr $NlMsgerr; +typedef struct rtgenmsg $RtGenmsg; +typedef struct nlattr $NlAttr; +typedef struct rtattr $RtAttr; +typedef struct ifinfomsg $IfInfomsg; +typedef struct ifaddrmsg $IfAddrmsg; +typedef struct rtmsg $RtMsg; +typedef struct rtnexthop $RtNexthop; // Inotify + typedef struct inotify_event $InotifyEvent; enum { $SizeofInotifyEvent = sizeof(struct inotify_event) }; - // Ptrace // Register structures diff --git a/src/pkg/syscall/zerrors_darwin_386.go b/src/pkg/syscall/zerrors_darwin_386.go index 7bc1280d6..5ee64ee70 100644 --- a/src/pkg/syscall/zerrors_darwin_386.go +++ b/src/pkg/syscall/zerrors_darwin_386.go @@ -339,6 +339,72 @@ const ( IFF_SIMPLEX = 0x800 IFF_UP = 0x1 IFNAMSIZ = 0x10 + IFT_1822 = 0x2 + IFT_AAL5 = 0x31 + IFT_ARCNET = 0x23 + IFT_ARCNETPLUS = 0x24 + IFT_ATM = 0x25 + IFT_BRIDGE = 0xd1 + IFT_CARP = 0xf8 + IFT_CEPT = 0x13 + IFT_DS3 = 0x1e + IFT_ENC = 0xf4 + IFT_EON = 0x19 + IFT_ETHER = 0x6 + IFT_FAITH = 0x38 + IFT_FDDI = 0xf + IFT_FRELAY = 0x20 + IFT_FRELAYDCE = 0x2c + IFT_GIF = 0x37 + IFT_HDH1822 = 0x3 + IFT_HIPPI = 0x2f + IFT_HSSI = 0x2e + IFT_HY = 0xe + IFT_IEEE1394 = 0x90 + IFT_IEEE8023ADLAG = 0x88 + IFT_ISDNBASIC = 0x14 + IFT_ISDNPRIMARY = 0x15 + IFT_ISO88022LLC = 0x29 + IFT_ISO88023 = 0x7 + IFT_ISO88024 = 0x8 + IFT_ISO88025 = 0x9 + IFT_ISO88026 = 0xa + IFT_L2VLAN = 0x87 + IFT_LAPB = 0x10 + IFT_LOCALTALK = 0x2a + IFT_LOOP = 0x18 + IFT_MIOX25 = 0x26 + IFT_MODEM = 0x30 + IFT_NSIP = 0x1b + IFT_OTHER = 0x1 + IFT_P10 = 0xc + IFT_P80 = 0xd + IFT_PARA = 0x22 + IFT_PDP = 0xff + IFT_PFLOG = 0xf5 + IFT_PFSYNC = 0xf6 + IFT_PPP = 0x17 + IFT_PROPMUX = 0x36 + IFT_PROPVIRTUAL = 0x35 + IFT_PTPSERIAL = 0x16 + IFT_RS232 = 0x21 + IFT_SDLC = 0x11 + IFT_SIP = 0x1f + IFT_SLIP = 0x1c + IFT_SMDSDXI = 0x2b + IFT_SMDSICIP = 0x34 + IFT_SONET = 0x27 + IFT_SONETPATH = 0x32 + IFT_SONETVT = 0x33 + IFT_STARLAN = 0xb + IFT_STF = 0x39 + IFT_T1 = 0x12 + IFT_ULTRA = 0x1d + IFT_V35 = 0x2d + IFT_X25 = 0x5 + IFT_X25DDN = 0x4 + IFT_X25PLE = 0x28 + IFT_XETHER = 0x1a IN_CLASSA_HOST = 0xffffff IN_CLASSA_MAX = 0x80 IN_CLASSA_NET = 0xff000000 diff --git a/src/pkg/syscall/zerrors_darwin_amd64.go b/src/pkg/syscall/zerrors_darwin_amd64.go index d76f09220..65a48d6e7 100644 --- a/src/pkg/syscall/zerrors_darwin_amd64.go +++ b/src/pkg/syscall/zerrors_darwin_amd64.go @@ -339,6 +339,72 @@ const ( IFF_SIMPLEX = 0x800 IFF_UP = 0x1 IFNAMSIZ = 0x10 + IFT_1822 = 0x2 + IFT_AAL5 = 0x31 + IFT_ARCNET = 0x23 + IFT_ARCNETPLUS = 0x24 + IFT_ATM = 0x25 + IFT_BRIDGE = 0xd1 + IFT_CARP = 0xf8 + IFT_CEPT = 0x13 + IFT_DS3 = 0x1e + IFT_ENC = 0xf4 + IFT_EON = 0x19 + IFT_ETHER = 0x6 + IFT_FAITH = 0x38 + IFT_FDDI = 0xf + IFT_FRELAY = 0x20 + IFT_FRELAYDCE = 0x2c + IFT_GIF = 0x37 + IFT_HDH1822 = 0x3 + IFT_HIPPI = 0x2f + IFT_HSSI = 0x2e + IFT_HY = 0xe + IFT_IEEE1394 = 0x90 + IFT_IEEE8023ADLAG = 0x88 + IFT_ISDNBASIC = 0x14 + IFT_ISDNPRIMARY = 0x15 + IFT_ISO88022LLC = 0x29 + IFT_ISO88023 = 0x7 + IFT_ISO88024 = 0x8 + IFT_ISO88025 = 0x9 + IFT_ISO88026 = 0xa + IFT_L2VLAN = 0x87 + IFT_LAPB = 0x10 + IFT_LOCALTALK = 0x2a + IFT_LOOP = 0x18 + IFT_MIOX25 = 0x26 + IFT_MODEM = 0x30 + IFT_NSIP = 0x1b + IFT_OTHER = 0x1 + IFT_P10 = 0xc + IFT_P80 = 0xd + IFT_PARA = 0x22 + IFT_PDP = 0xff + IFT_PFLOG = 0xf5 + IFT_PFSYNC = 0xf6 + IFT_PPP = 0x17 + IFT_PROPMUX = 0x36 + IFT_PROPVIRTUAL = 0x35 + IFT_PTPSERIAL = 0x16 + IFT_RS232 = 0x21 + IFT_SDLC = 0x11 + IFT_SIP = 0x1f + IFT_SLIP = 0x1c + IFT_SMDSDXI = 0x2b + IFT_SMDSICIP = 0x34 + IFT_SONET = 0x27 + IFT_SONETPATH = 0x32 + IFT_SONETVT = 0x33 + IFT_STARLAN = 0xb + IFT_STF = 0x39 + IFT_T1 = 0x12 + IFT_ULTRA = 0x1d + IFT_V35 = 0x2d + IFT_X25 = 0x5 + IFT_X25DDN = 0x4 + IFT_X25PLE = 0x28 + IFT_XETHER = 0x1a IN_CLASSA_HOST = 0xffffff IN_CLASSA_MAX = 0x80 IN_CLASSA_NET = 0xff000000 diff --git a/src/pkg/syscall/zerrors_freebsd_386.go b/src/pkg/syscall/zerrors_freebsd_386.go index 9c21c71eb..52e42487b 100644 --- a/src/pkg/syscall/zerrors_freebsd_386.go +++ b/src/pkg/syscall/zerrors_freebsd_386.go @@ -9,1013 +9,1228 @@ package syscall // Constants const ( - AF_APPLETALK = 0x10 - AF_ARP = 0x23 - AF_ATM = 0x1e - AF_BLUETOOTH = 0x24 - AF_CCITT = 0xa - AF_CHAOS = 0x5 - AF_CNT = 0x15 - AF_COIP = 0x14 - AF_DATAKIT = 0x9 - AF_DECnet = 0xc - AF_DLI = 0xd - AF_E164 = 0x1a - AF_ECMA = 0x8 - AF_HYLINK = 0xf - AF_IEEE80211 = 0x25 - AF_IMPLINK = 0x3 - AF_INET = 0x2 - AF_INET6 = 0x1c - AF_IPX = 0x17 - AF_ISDN = 0x1a - AF_ISO = 0x7 - AF_LAT = 0xe - AF_LINK = 0x12 - AF_LOCAL = 0x1 - AF_MAX = 0x26 - AF_NATM = 0x1d - AF_NETBIOS = 0x6 - AF_NETGRAPH = 0x20 - AF_OSI = 0x7 - AF_PUP = 0x4 - AF_ROUTE = 0x11 - AF_SCLUSTER = 0x22 - AF_SIP = 0x18 - AF_SLOW = 0x21 - AF_SNA = 0xb - AF_UNIX = 0x1 - AF_UNSPEC = 0 - AF_VENDOR00 = 0x27 - AF_VENDOR01 = 0x29 - AF_VENDOR02 = 0x2b - AF_VENDOR03 = 0x2d - AF_VENDOR04 = 0x2f - AF_VENDOR05 = 0x31 - AF_VENDOR06 = 0x33 - AF_VENDOR07 = 0x35 - AF_VENDOR08 = 0x37 - AF_VENDOR09 = 0x39 - AF_VENDOR10 = 0x3b - AF_VENDOR11 = 0x3d - AF_VENDOR12 = 0x3f - AF_VENDOR13 = 0x41 - AF_VENDOR14 = 0x43 - AF_VENDOR15 = 0x45 - AF_VENDOR16 = 0x47 - AF_VENDOR17 = 0x49 - AF_VENDOR18 = 0x4b - AF_VENDOR19 = 0x4d - AF_VENDOR20 = 0x4f - AF_VENDOR21 = 0x51 - AF_VENDOR22 = 0x53 - AF_VENDOR23 = 0x55 - AF_VENDOR24 = 0x57 - AF_VENDOR25 = 0x59 - AF_VENDOR26 = 0x5b - AF_VENDOR27 = 0x5d - AF_VENDOR28 = 0x5f - AF_VENDOR29 = 0x61 - AF_VENDOR30 = 0x63 - AF_VENDOR31 = 0x65 - AF_VENDOR32 = 0x67 - AF_VENDOR33 = 0x69 - AF_VENDOR34 = 0x6b - AF_VENDOR35 = 0x6d - AF_VENDOR36 = 0x6f - AF_VENDOR37 = 0x71 - AF_VENDOR38 = 0x73 - AF_VENDOR39 = 0x75 - AF_VENDOR40 = 0x77 - AF_VENDOR41 = 0x79 - AF_VENDOR42 = 0x7b - AF_VENDOR43 = 0x7d - AF_VENDOR44 = 0x7f - AF_VENDOR45 = 0x81 - AF_VENDOR46 = 0x83 - AF_VENDOR47 = 0x85 - BIOCFEEDBACK = 0x8004427c - BIOCFLUSH = 0x20004268 - BIOCGBLEN = 0x40044266 - BIOCGDIRECTION = 0x40044276 - BIOCGDLT = 0x4004426a - BIOCGDLTLIST = 0xc0084279 - BIOCGETBUFMODE = 0x4004427d - BIOCGETIF = 0x4020426b - BIOCGETZMAX = 0x4004427f - BIOCGHDRCMPLT = 0x40044274 - BIOCGRSIG = 0x40044272 - BIOCGRTIMEOUT = 0x4008426e - BIOCGSEESENT = 0x40044276 - BIOCGSTATS = 0x4008426f - BIOCIMMEDIATE = 0x80044270 - BIOCLOCK = 0x2000427a - BIOCPROMISC = 0x20004269 - BIOCROTZBUF = 0x400c4280 - BIOCSBLEN = 0xc0044266 - BIOCSDIRECTION = 0x80044277 - BIOCSDLT = 0x80044278 - BIOCSETBUFMODE = 0x8004427e - BIOCSETF = 0x80084267 - BIOCSETFNR = 0x80084282 - BIOCSETIF = 0x8020426c - BIOCSETWF = 0x8008427b - BIOCSETZBUF = 0x800c4281 - BIOCSHDRCMPLT = 0x80044275 - BIOCSRSIG = 0x80044273 - BIOCSRTIMEOUT = 0x8008426d - BIOCSSEESENT = 0x80044277 - BIOCVERSION = 0x40044271 - BPF_A = 0x10 - BPF_ABS = 0x20 - BPF_ADD = 0 - BPF_ALIGNMENT = 0x4 - BPF_ALU = 0x4 - BPF_AND = 0x50 - BPF_B = 0x10 - BPF_BUFMODE_BUFFER = 0x1 - BPF_BUFMODE_ZBUF = 0x2 - BPF_DIV = 0x30 - BPF_H = 0x8 - BPF_IMM = 0 - BPF_IND = 0x40 - BPF_JA = 0 - BPF_JEQ = 0x10 - BPF_JGE = 0x30 - BPF_JGT = 0x20 - BPF_JMP = 0x5 - BPF_JSET = 0x40 - BPF_K = 0 - BPF_LD = 0 - BPF_LDX = 0x1 - BPF_LEN = 0x80 - BPF_LSH = 0x60 - BPF_MAJOR_VERSION = 0x1 - BPF_MAXBUFSIZE = 0x80000 - BPF_MAXINSNS = 0x200 - BPF_MEM = 0x60 - BPF_MEMWORDS = 0x10 - BPF_MINBUFSIZE = 0x20 - BPF_MINOR_VERSION = 0x1 - BPF_MISC = 0x7 - BPF_MSH = 0xa0 - BPF_MUL = 0x20 - BPF_NEG = 0x80 - BPF_OR = 0x40 - BPF_RELEASE = 0x30bb6 - BPF_RET = 0x6 - BPF_RSH = 0x70 - BPF_ST = 0x2 - BPF_STX = 0x3 - BPF_SUB = 0x10 - BPF_TAX = 0 - BPF_TXA = 0x80 - BPF_W = 0 - BPF_X = 0x8 - CTL_MAXNAME = 0x18 - CTL_NET = 0x4 - DLT_A429 = 0xb8 - DLT_A653_ICM = 0xb9 - DLT_AIRONET_HEADER = 0x78 - DLT_APPLE_IP_OVER_IEEE1394 = 0x8a - DLT_ARCNET = 0x7 - DLT_ARCNET_LINUX = 0x81 - DLT_ATM_CLIP = 0x13 - DLT_ATM_RFC1483 = 0xb - DLT_AURORA = 0x7e - DLT_AX25 = 0x3 - DLT_AX25_KISS = 0xca - DLT_BACNET_MS_TP = 0xa5 - DLT_BLUETOOTH_HCI_H4 = 0xbb - DLT_BLUETOOTH_HCI_H4_WITH_PHDR = 0xc9 - DLT_CAN20B = 0xbe - DLT_CHAOS = 0x5 - DLT_CHDLC = 0x68 - DLT_CISCO_IOS = 0x76 - DLT_C_HDLC = 0x68 - DLT_C_HDLC_WITH_DIR = 0xcd - DLT_DOCSIS = 0x8f - DLT_ECONET = 0x73 - DLT_EN10MB = 0x1 - DLT_EN3MB = 0x2 - DLT_ENC = 0x6d - DLT_ERF = 0xc5 - DLT_ERF_ETH = 0xaf - DLT_ERF_POS = 0xb0 - DLT_FDDI = 0xa - DLT_FLEXRAY = 0xd2 - DLT_FRELAY = 0x6b - DLT_FRELAY_WITH_DIR = 0xce - DLT_GCOM_SERIAL = 0xad - DLT_GCOM_T1E1 = 0xac - DLT_GPF_F = 0xab - DLT_GPF_T = 0xaa - DLT_GPRS_LLC = 0xa9 - DLT_HHDLC = 0x79 - DLT_IBM_SN = 0x92 - DLT_IBM_SP = 0x91 - DLT_IEEE802 = 0x6 - DLT_IEEE802_11 = 0x69 - DLT_IEEE802_11_RADIO = 0x7f - DLT_IEEE802_11_RADIO_AVS = 0xa3 - DLT_IEEE802_15_4 = 0xc3 - DLT_IEEE802_15_4_LINUX = 0xbf - DLT_IEEE802_15_4_NONASK_PHY = 0xd7 - DLT_IEEE802_16_MAC_CPS = 0xbc - DLT_IEEE802_16_MAC_CPS_RADIO = 0xc1 - DLT_IPFILTER = 0x74 - DLT_IPMB = 0xc7 - DLT_IPMB_LINUX = 0xd1 - DLT_IP_OVER_FC = 0x7a - DLT_JUNIPER_ATM1 = 0x89 - DLT_JUNIPER_ATM2 = 0x87 - DLT_JUNIPER_CHDLC = 0xb5 - DLT_JUNIPER_ES = 0x84 - DLT_JUNIPER_ETHER = 0xb2 - DLT_JUNIPER_FRELAY = 0xb4 - DLT_JUNIPER_GGSN = 0x85 - DLT_JUNIPER_ISM = 0xc2 - DLT_JUNIPER_MFR = 0x86 - DLT_JUNIPER_MLFR = 0x83 - DLT_JUNIPER_MLPPP = 0x82 - DLT_JUNIPER_MONITOR = 0xa4 - DLT_JUNIPER_PIC_PEER = 0xae - DLT_JUNIPER_PPP = 0xb3 - DLT_JUNIPER_PPPOE = 0xa7 - DLT_JUNIPER_PPPOE_ATM = 0xa8 - DLT_JUNIPER_SERVICES = 0x88 - DLT_JUNIPER_ST = 0xc8 - DLT_JUNIPER_VP = 0xb7 - DLT_LAPB_WITH_DIR = 0xcf - DLT_LAPD = 0xcb - DLT_LIN = 0xd4 - DLT_LINUX_IRDA = 0x90 - DLT_LINUX_LAPD = 0xb1 - DLT_LINUX_PPP_WITHDIRECTION = 0xa6 - DLT_LINUX_SLL = 0x71 - DLT_LOOP = 0x6c - DLT_LTALK = 0x72 - DLT_MFR = 0xb6 - DLT_MOST = 0xd3 - DLT_MTP2 = 0x8c - DLT_MTP2_WITH_PHDR = 0x8b - DLT_MTP3 = 0x8d - DLT_NULL = 0 - DLT_PCI_EXP = 0x7d - DLT_PFLOG = 0x75 - DLT_PFSYNC = 0x79 - DLT_PPI = 0xc0 - DLT_PPP = 0x9 - DLT_PPP_BSDOS = 0x10 - DLT_PPP_ETHER = 0x33 - DLT_PPP_PPPD = 0xa6 - DLT_PPP_SERIAL = 0x32 - DLT_PPP_WITH_DIR = 0xcc - DLT_PPP_WITH_DIRECTION = 0xa6 - DLT_PRISM_HEADER = 0x77 - DLT_PRONET = 0x4 - DLT_RAIF1 = 0xc6 - DLT_RAW = 0xc - DLT_RIO = 0x7c - DLT_SCCP = 0x8e - DLT_SITA = 0xc4 - DLT_SLIP = 0x8 - DLT_SLIP_BSDOS = 0xf - DLT_SUNATM = 0x7b - DLT_SYMANTEC_FIREWALL = 0x63 - DLT_TZSP = 0x80 - DLT_USB = 0xba - DLT_USB_LINUX = 0xbd - DLT_USER0 = 0x93 - DLT_USER1 = 0x94 - DLT_USER10 = 0x9d - DLT_USER11 = 0x9e - DLT_USER12 = 0x9f - DLT_USER13 = 0xa0 - DLT_USER14 = 0xa1 - DLT_USER15 = 0xa2 - DLT_USER2 = 0x95 - DLT_USER3 = 0x96 - DLT_USER4 = 0x97 - DLT_USER5 = 0x98 - DLT_USER6 = 0x99 - DLT_USER7 = 0x9a - DLT_USER8 = 0x9b - DLT_USER9 = 0x9c - DLT_X2E_SERIAL = 0xd5 - DLT_X2E_XORAYA = 0xd6 - E2BIG = 0x7 - EACCES = 0xd - EADDRINUSE = 0x30 - EADDRNOTAVAIL = 0x31 - EAFNOSUPPORT = 0x2f - EAGAIN = 0x23 - EALREADY = 0x25 - EAUTH = 0x50 - EBADF = 0x9 - EBADMSG = 0x59 - EBADRPC = 0x48 - EBUSY = 0x10 - ECANCELED = 0x55 - ECHILD = 0xa - ECONNABORTED = 0x35 - ECONNREFUSED = 0x3d - ECONNRESET = 0x36 - EDEADLK = 0xb - EDESTADDRREQ = 0x27 - EDOM = 0x21 - EDOOFUS = 0x58 - EDQUOT = 0x45 - EEXIST = 0x11 - EFAULT = 0xe - EFBIG = 0x1b - EFTYPE = 0x4f - EHOSTDOWN = 0x40 - EHOSTUNREACH = 0x41 - EIDRM = 0x52 - EILSEQ = 0x56 - EINPROGRESS = 0x24 - EINTR = 0x4 - EINVAL = 0x16 - EIO = 0x5 - EISCONN = 0x38 - EISDIR = 0x15 - ELAST = 0x5d - ELOOP = 0x3e - EMFILE = 0x18 - EMLINK = 0x1f - EMSGSIZE = 0x28 - EMULTIHOP = 0x5a - ENAMETOOLONG = 0x3f - ENEEDAUTH = 0x51 - ENETDOWN = 0x32 - ENETRESET = 0x34 - ENETUNREACH = 0x33 - ENFILE = 0x17 - ENOATTR = 0x57 - ENOBUFS = 0x37 - ENODEV = 0x13 - ENOENT = 0x2 - ENOEXEC = 0x8 - ENOLCK = 0x4d - ENOLINK = 0x5b - ENOMEM = 0xc - ENOMSG = 0x53 - ENOPROTOOPT = 0x2a - ENOSPC = 0x1c - ENOSYS = 0x4e - ENOTBLK = 0xf - ENOTCAPABLE = 0x5d - ENOTCONN = 0x39 - ENOTDIR = 0x14 - ENOTEMPTY = 0x42 - ENOTSOCK = 0x26 - ENOTSUP = 0x2d - ENOTTY = 0x19 - ENXIO = 0x6 - EOPNOTSUPP = 0x2d - EOVERFLOW = 0x54 - EPERM = 0x1 - EPFNOSUPPORT = 0x2e - EPIPE = 0x20 - EPROCLIM = 0x43 - EPROCUNAVAIL = 0x4c - EPROGMISMATCH = 0x4b - EPROGUNAVAIL = 0x4a - EPROTO = 0x5c - EPROTONOSUPPORT = 0x2b - EPROTOTYPE = 0x29 - ERANGE = 0x22 - EREMOTE = 0x47 - EROFS = 0x1e - ERPCMISMATCH = 0x49 - ESHUTDOWN = 0x3a - ESOCKTNOSUPPORT = 0x2c - ESPIPE = 0x1d - ESRCH = 0x3 - ESTALE = 0x46 - ETIMEDOUT = 0x3c - ETOOMANYREFS = 0x3b - ETXTBSY = 0x1a - EUSERS = 0x44 - EVFILT_AIO = -0x3 - EVFILT_FS = -0x9 - EVFILT_LIO = -0xa - EVFILT_PROC = -0x5 - EVFILT_READ = -0x1 - EVFILT_SIGNAL = -0x6 - EVFILT_SYSCOUNT = 0xb - EVFILT_TIMER = -0x7 - EVFILT_USER = -0xb - EVFILT_VNODE = -0x4 - EVFILT_WRITE = -0x2 - EV_ADD = 0x1 - EV_CLEAR = 0x20 - EV_DELETE = 0x2 - EV_DISABLE = 0x8 - EV_DISPATCH = 0x80 - EV_ENABLE = 0x4 - EV_EOF = 0x8000 - EV_ERROR = 0x4000 - EV_FLAG1 = 0x2000 - EV_ONESHOT = 0x10 - EV_RECEIPT = 0x40 - EV_SYSFLAGS = 0xf000 - EWOULDBLOCK = 0x23 - EXDEV = 0x12 - FD_CLOEXEC = 0x1 - FD_SETSIZE = 0x400 - F_CANCEL = 0x5 - F_DUP2FD = 0xa - F_DUPFD = 0 - F_GETFD = 0x1 - F_GETFL = 0x3 - F_GETLK = 0xb - F_GETOWN = 0x5 - F_OGETLK = 0x7 - F_OSETLK = 0x8 - F_OSETLKW = 0x9 - F_RDAHEAD = 0x10 - F_RDLCK = 0x1 - F_READAHEAD = 0xf - F_SETFD = 0x2 - F_SETFL = 0x4 - F_SETLK = 0xc - F_SETLKW = 0xd - F_SETLK_REMOTE = 0xe - F_SETOWN = 0x6 - F_UNLCK = 0x2 - F_UNLCKSYS = 0x4 - F_WRLCK = 0x3 - IFF_ALLMULTI = 0x200 - IFF_ALTPHYS = 0x4000 - IFF_BROADCAST = 0x2 - IFF_CANTCHANGE = 0x208f72 - IFF_DEBUG = 0x4 - IFF_DRV_OACTIVE = 0x400 - IFF_DRV_RUNNING = 0x40 - IFF_DYING = 0x200000 - IFF_LINK0 = 0x1000 - IFF_LINK1 = 0x2000 - IFF_LINK2 = 0x4000 - IFF_LOOPBACK = 0x8 - IFF_MONITOR = 0x40000 - IFF_MULTICAST = 0x8000 - IFF_NOARP = 0x80 - IFF_OACTIVE = 0x400 - IFF_POINTOPOINT = 0x10 - IFF_PPROMISC = 0x20000 - IFF_PROMISC = 0x100 - IFF_RENAMING = 0x400000 - IFF_RUNNING = 0x40 - IFF_SIMPLEX = 0x800 - IFF_SMART = 0x20 - IFF_STATICARP = 0x80000 - IFF_UP = 0x1 - IFNAMSIZ = 0x10 - IN_CLASSA_HOST = 0xffffff - IN_CLASSA_MAX = 0x80 - IN_CLASSA_NET = 0xff000000 - IN_CLASSA_NSHIFT = 0x18 - IN_CLASSB_HOST = 0xffff - IN_CLASSB_MAX = 0x10000 - IN_CLASSB_NET = 0xffff0000 - IN_CLASSB_NSHIFT = 0x10 - IN_CLASSC_HOST = 0xff - IN_CLASSC_NET = 0xffffff00 - IN_CLASSC_NSHIFT = 0x8 - IN_CLASSD_HOST = 0xfffffff - IN_CLASSD_NET = 0xf0000000 - IN_CLASSD_NSHIFT = 0x1c - IN_LOOPBACKNET = 0x7f - IPPROTO_3PC = 0x22 - IPPROTO_ADFS = 0x44 - IPPROTO_AH = 0x33 - IPPROTO_AHIP = 0x3d - IPPROTO_APES = 0x63 - IPPROTO_ARGUS = 0xd - IPPROTO_AX25 = 0x5d - IPPROTO_BHA = 0x31 - IPPROTO_BLT = 0x1e - IPPROTO_BRSATMON = 0x4c - IPPROTO_CARP = 0x70 - IPPROTO_CFTP = 0x3e - IPPROTO_CHAOS = 0x10 - IPPROTO_CMTP = 0x26 - IPPROTO_CPHB = 0x49 - IPPROTO_CPNX = 0x48 - IPPROTO_DDP = 0x25 - IPPROTO_DGP = 0x56 - IPPROTO_DIVERT = 0x102 - IPPROTO_DONE = 0x101 - IPPROTO_DSTOPTS = 0x3c - IPPROTO_EGP = 0x8 - IPPROTO_EMCON = 0xe - IPPROTO_ENCAP = 0x62 - IPPROTO_EON = 0x50 - IPPROTO_ESP = 0x32 - IPPROTO_ETHERIP = 0x61 - IPPROTO_FRAGMENT = 0x2c - IPPROTO_GGP = 0x3 - IPPROTO_GMTP = 0x64 - IPPROTO_GRE = 0x2f - IPPROTO_HELLO = 0x3f - IPPROTO_HMP = 0x14 - IPPROTO_HOPOPTS = 0 - IPPROTO_ICMP = 0x1 - IPPROTO_ICMPV6 = 0x3a - IPPROTO_IDP = 0x16 - IPPROTO_IDPR = 0x23 - IPPROTO_IDRP = 0x2d - IPPROTO_IGMP = 0x2 - IPPROTO_IGP = 0x55 - IPPROTO_IGRP = 0x58 - IPPROTO_IL = 0x28 - IPPROTO_INLSP = 0x34 - IPPROTO_INP = 0x20 - IPPROTO_IP = 0 - IPPROTO_IPCOMP = 0x6c - IPPROTO_IPCV = 0x47 - IPPROTO_IPEIP = 0x5e - IPPROTO_IPIP = 0x4 - IPPROTO_IPPC = 0x43 - IPPROTO_IPV4 = 0x4 - IPPROTO_IPV6 = 0x29 - IPPROTO_IRTP = 0x1c - IPPROTO_KRYPTOLAN = 0x41 - IPPROTO_LARP = 0x5b - IPPROTO_LEAF1 = 0x19 - IPPROTO_LEAF2 = 0x1a - IPPROTO_MAX = 0x100 - IPPROTO_MAXID = 0x34 - IPPROTO_MEAS = 0x13 - IPPROTO_MHRP = 0x30 - IPPROTO_MICP = 0x5f - IPPROTO_MOBILE = 0x37 - IPPROTO_MTP = 0x5c - IPPROTO_MUX = 0x12 - IPPROTO_ND = 0x4d - IPPROTO_NHRP = 0x36 - IPPROTO_NONE = 0x3b - IPPROTO_NSP = 0x1f - IPPROTO_NVPII = 0xb - IPPROTO_OLD_DIVERT = 0xfe - IPPROTO_OSPFIGP = 0x59 - IPPROTO_PFSYNC = 0xf0 - IPPROTO_PGM = 0x71 - IPPROTO_PIGP = 0x9 - IPPROTO_PIM = 0x67 - IPPROTO_PRM = 0x15 - IPPROTO_PUP = 0xc - IPPROTO_PVP = 0x4b - IPPROTO_RAW = 0xff - IPPROTO_RCCMON = 0xa - IPPROTO_RDP = 0x1b - IPPROTO_ROUTING = 0x2b - IPPROTO_RSVP = 0x2e - IPPROTO_RVD = 0x42 - IPPROTO_SATEXPAK = 0x40 - IPPROTO_SATMON = 0x45 - IPPROTO_SCCSP = 0x60 - IPPROTO_SCTP = 0x84 - IPPROTO_SDRP = 0x2a - IPPROTO_SEP = 0x21 - IPPROTO_SKIP = 0x39 - IPPROTO_SPACER = 0x7fff - IPPROTO_SRPC = 0x5a - IPPROTO_ST = 0x7 - IPPROTO_SVMTP = 0x52 - IPPROTO_SWIPE = 0x35 - IPPROTO_TCF = 0x57 - IPPROTO_TCP = 0x6 - IPPROTO_TLSP = 0x38 - IPPROTO_TP = 0x1d - IPPROTO_TPXX = 0x27 - IPPROTO_TRUNK1 = 0x17 - IPPROTO_TRUNK2 = 0x18 - IPPROTO_TTP = 0x54 - IPPROTO_UDP = 0x11 - IPPROTO_VINES = 0x53 - IPPROTO_VISA = 0x46 - IPPROTO_VMTP = 0x51 - IPPROTO_WBEXPAK = 0x4f - IPPROTO_WBMON = 0x4e - IPPROTO_WSN = 0x4a - IPPROTO_XNET = 0xf - IPPROTO_XTP = 0x24 - IPV6_AUTOFLOWLABEL = 0x3b - IPV6_BINDANY = 0x40 - IPV6_BINDV6ONLY = 0x1b - IPV6_CHECKSUM = 0x1a - IPV6_DEFAULT_MULTICAST_HOPS = 0x1 - IPV6_DEFAULT_MULTICAST_LOOP = 0x1 - IPV6_DEFHLIM = 0x40 - IPV6_DONTFRAG = 0x3e - IPV6_DSTOPTS = 0x32 - IPV6_FAITH = 0x1d - IPV6_FLOWINFO_MASK = 0xffffff0f - IPV6_FLOWLABEL_MASK = 0xffff0f00 - IPV6_FRAGTTL = 0x78 - IPV6_FW_ADD = 0x1e - IPV6_FW_DEL = 0x1f - IPV6_FW_FLUSH = 0x20 - IPV6_FW_GET = 0x22 - IPV6_FW_ZERO = 0x21 - IPV6_HLIMDEC = 0x1 - IPV6_HOPLIMIT = 0x2f - IPV6_HOPOPTS = 0x31 - IPV6_IPSEC_POLICY = 0x1c - IPV6_JOIN_GROUP = 0xc - IPV6_LEAVE_GROUP = 0xd - IPV6_MAXHLIM = 0xff - IPV6_MAXOPTHDR = 0x800 - IPV6_MAXPACKET = 0xffff - IPV6_MAX_GROUP_SRC_FILTER = 0x200 - IPV6_MAX_MEMBERSHIPS = 0xfff - IPV6_MAX_SOCK_SRC_FILTER = 0x80 - IPV6_MIN_MEMBERSHIPS = 0x1f - IPV6_MMTU = 0x500 - IPV6_MSFILTER = 0x4a - IPV6_MULTICAST_HOPS = 0xa - IPV6_MULTICAST_IF = 0x9 - IPV6_MULTICAST_LOOP = 0xb - IPV6_NEXTHOP = 0x30 - IPV6_PATHMTU = 0x2c - IPV6_PKTINFO = 0x2e - IPV6_PORTRANGE = 0xe - IPV6_PORTRANGE_DEFAULT = 0 - IPV6_PORTRANGE_HIGH = 0x1 - IPV6_PORTRANGE_LOW = 0x2 - IPV6_PREFER_TEMPADDR = 0x3f - IPV6_RECVDSTOPTS = 0x28 - IPV6_RECVHOPLIMIT = 0x25 - IPV6_RECVHOPOPTS = 0x27 - IPV6_RECVPATHMTU = 0x2b - IPV6_RECVPKTINFO = 0x24 - IPV6_RECVRTHDR = 0x26 - IPV6_RECVTCLASS = 0x39 - IPV6_RTHDR = 0x33 - IPV6_RTHDRDSTOPTS = 0x23 - IPV6_RTHDR_LOOSE = 0 - IPV6_RTHDR_STRICT = 0x1 - IPV6_RTHDR_TYPE_0 = 0 - IPV6_SOCKOPT_RESERVED1 = 0x3 - IPV6_TCLASS = 0x3d - IPV6_UNICAST_HOPS = 0x4 - IPV6_USE_MIN_MTU = 0x2a - IPV6_V6ONLY = 0x1b - IPV6_VERSION = 0x60 - IPV6_VERSION_MASK = 0xf0 - IP_ADD_MEMBERSHIP = 0xc - IP_ADD_SOURCE_MEMBERSHIP = 0x46 - IP_BINDANY = 0x18 - IP_BLOCK_SOURCE = 0x48 - IP_DEFAULT_MULTICAST_LOOP = 0x1 - IP_DEFAULT_MULTICAST_TTL = 0x1 - IP_DF = 0x4000 - IP_DONTFRAG = 0x43 - IP_DROP_MEMBERSHIP = 0xd - IP_DROP_SOURCE_MEMBERSHIP = 0x47 - IP_DUMMYNET3 = 0x31 - IP_DUMMYNET_CONFIGURE = 0x3c - IP_DUMMYNET_DEL = 0x3d - IP_DUMMYNET_FLUSH = 0x3e - IP_DUMMYNET_GET = 0x40 - IP_FAITH = 0x16 - IP_FW3 = 0x30 - IP_FW_ADD = 0x32 - IP_FW_DEL = 0x33 - IP_FW_FLUSH = 0x34 - IP_FW_GET = 0x36 - IP_FW_NAT_CFG = 0x38 - IP_FW_NAT_DEL = 0x39 - IP_FW_NAT_GET_CONFIG = 0x3a - IP_FW_NAT_GET_LOG = 0x3b - IP_FW_RESETLOG = 0x37 - IP_FW_TABLE_ADD = 0x28 - IP_FW_TABLE_DEL = 0x29 - IP_FW_TABLE_FLUSH = 0x2a - IP_FW_TABLE_GETSIZE = 0x2b - IP_FW_TABLE_LIST = 0x2c - IP_FW_ZERO = 0x35 - IP_HDRINCL = 0x2 - IP_IPSEC_POLICY = 0x15 - IP_MAXPACKET = 0xffff - IP_MAX_GROUP_SRC_FILTER = 0x200 - IP_MAX_MEMBERSHIPS = 0xfff - IP_MAX_SOCK_MUTE_FILTER = 0x80 - IP_MAX_SOCK_SRC_FILTER = 0x80 - IP_MAX_SOURCE_FILTER = 0x400 - IP_MF = 0x2000 - IP_MINTTL = 0x42 - IP_MIN_MEMBERSHIPS = 0x1f - IP_MSFILTER = 0x4a - IP_MSS = 0x240 - IP_MULTICAST_IF = 0x9 - IP_MULTICAST_LOOP = 0xb - IP_MULTICAST_TTL = 0xa - IP_MULTICAST_VIF = 0xe - IP_OFFMASK = 0x1fff - IP_ONESBCAST = 0x17 - IP_OPTIONS = 0x1 - IP_PORTRANGE = 0x13 - IP_PORTRANGE_DEFAULT = 0 - IP_PORTRANGE_HIGH = 0x1 - IP_PORTRANGE_LOW = 0x2 - IP_RECVDSTADDR = 0x7 - IP_RECVIF = 0x14 - IP_RECVOPTS = 0x5 - IP_RECVRETOPTS = 0x6 - IP_RECVTTL = 0x41 - IP_RETOPTS = 0x8 - IP_RF = 0x8000 - IP_RSVP_OFF = 0x10 - IP_RSVP_ON = 0xf - IP_RSVP_VIF_OFF = 0x12 - IP_RSVP_VIF_ON = 0x11 - IP_SENDSRCADDR = 0x7 - IP_TOS = 0x3 - IP_TTL = 0x4 - IP_UNBLOCK_SOURCE = 0x49 - MSG_COMPAT = 0x8000 - MSG_CTRUNC = 0x20 - MSG_DONTROUTE = 0x4 - MSG_DONTWAIT = 0x80 - MSG_EOF = 0x100 - MSG_EOR = 0x8 - MSG_NBIO = 0x4000 - MSG_NOSIGNAL = 0x20000 - MSG_NOTIFICATION = 0x2000 - MSG_OOB = 0x1 - MSG_PEEK = 0x2 - MSG_TRUNC = 0x10 - MSG_WAITALL = 0x40 - NET_RT_DUMP = 0x1 - NET_RT_FLAGS = 0x2 - NET_RT_IFLIST = 0x3 - NET_RT_IFMALIST = 0x4 - NET_RT_MAXID = 0x5 - O_ACCMODE = 0x3 - O_APPEND = 0x8 - O_ASYNC = 0x40 - O_CREAT = 0x200 - O_DIRECT = 0x10000 - O_DIRECTORY = 0x20000 - O_EXCL = 0x800 - O_EXEC = 0x40000 - O_EXLOCK = 0x20 - O_FSYNC = 0x80 - O_NDELAY = 0x4 - O_NOCTTY = 0x8000 - O_NOFOLLOW = 0x100 - O_NONBLOCK = 0x4 - O_RDONLY = 0 - O_RDWR = 0x2 - O_SHLOCK = 0x10 - O_SYNC = 0x80 - O_TRUNC = 0x400 - O_TTY_INIT = 0x80000 - O_WRONLY = 0x1 - RTAX_AUTHOR = 0x6 - RTAX_BRD = 0x7 - RTAX_DST = 0 - RTAX_GATEWAY = 0x1 - RTAX_GENMASK = 0x3 - RTAX_IFA = 0x5 - RTAX_IFP = 0x4 - RTAX_MAX = 0x8 - RTAX_NETMASK = 0x2 - RTA_AUTHOR = 0x40 - RTA_BRD = 0x80 - RTA_DST = 0x1 - RTA_GATEWAY = 0x2 - RTA_GENMASK = 0x8 - RTA_IFA = 0x20 - RTA_IFP = 0x10 - RTA_NETMASK = 0x4 - RTF_BLACKHOLE = 0x1000 - RTF_BROADCAST = 0x400000 - RTF_DONE = 0x40 - RTF_DYNAMIC = 0x10 - RTF_FMASK = 0x1004d808 - RTF_GATEWAY = 0x2 - RTF_HOST = 0x4 - RTF_LLDATA = 0x400 - RTF_LLINFO = 0x400 - RTF_LOCAL = 0x200000 - RTF_MODIFIED = 0x20 - RTF_MULTICAST = 0x800000 - RTF_PINNED = 0x100000 - RTF_PRCLONING = 0x10000 - RTF_PROTO1 = 0x8000 - RTF_PROTO2 = 0x4000 - RTF_PROTO3 = 0x40000 - RTF_REJECT = 0x8 - RTF_RNH_LOCKED = 0x40000000 - RTF_STATIC = 0x800 - RTF_STICKY = 0x10000000 - RTF_UP = 0x1 - RTF_XRESOLVE = 0x200 - RTM_ADD = 0x1 - RTM_CHANGE = 0x3 - RTM_DELADDR = 0xd - RTM_DELETE = 0x2 - RTM_DELMADDR = 0x10 - RTM_GET = 0x4 - RTM_IEEE80211 = 0x12 - RTM_IFANNOUNCE = 0x11 - RTM_IFINFO = 0xe - RTM_LOCK = 0x8 - RTM_LOSING = 0x5 - RTM_MISS = 0x7 - RTM_NEWADDR = 0xc - RTM_NEWMADDR = 0xf - RTM_OLDADD = 0x9 - RTM_OLDDEL = 0xa - RTM_REDIRECT = 0x6 - RTM_RESOLVE = 0xb - RTM_RTTUNIT = 0xf4240 - RTM_VERSION = 0x5 - RTV_EXPIRE = 0x4 - RTV_HOPCOUNT = 0x2 - RTV_MTU = 0x1 - RTV_RPIPE = 0x8 - RTV_RTT = 0x40 - RTV_RTTVAR = 0x80 - RTV_SPIPE = 0x10 - RTV_SSTHRESH = 0x20 - RTV_WEIGHT = 0x100 - SCM_BINTIME = 0x4 - SCM_CREDS = 0x3 - SCM_RIGHTS = 0x1 - SCM_TIMESTAMP = 0x2 - SHUT_RD = 0 - SHUT_RDWR = 0x2 - SHUT_WR = 0x1 - SIGABRT = 0x6 - SIGALRM = 0xe - SIGBUS = 0xa - SIGCHLD = 0x14 - SIGCONT = 0x13 - SIGEMT = 0x7 - SIGFPE = 0x8 - SIGHUP = 0x1 - SIGILL = 0x4 - SIGINFO = 0x1d - SIGINT = 0x2 - SIGIO = 0x17 - SIGIOT = 0x6 - SIGKILL = 0x9 - SIGLWP = 0x20 - SIGPIPE = 0xd - SIGPROF = 0x1b - SIGQUIT = 0x3 - SIGSEGV = 0xb - SIGSTOP = 0x11 - SIGSYS = 0xc - SIGTERM = 0xf - SIGTHR = 0x20 - SIGTRAP = 0x5 - SIGTSTP = 0x12 - SIGTTIN = 0x15 - SIGTTOU = 0x16 - SIGURG = 0x10 - SIGUSR1 = 0x1e - SIGUSR2 = 0x1f - SIGVTALRM = 0x1a - SIGWINCH = 0x1c - SIGXCPU = 0x18 - SIGXFSZ = 0x19 - SIOCADDMULTI = 0x80206931 - SIOCADDRT = 0x8030720a - SIOCAIFADDR = 0x8040691a - SIOCAIFGROUP = 0x80246987 - SIOCALIFADDR = 0x8118691b - SIOCATMARK = 0x40047307 - SIOCDELMULTI = 0x80206932 - SIOCDELRT = 0x8030720b - SIOCDIFADDR = 0x80206919 - SIOCDIFGROUP = 0x80246989 - SIOCDIFPHYADDR = 0x80206949 - SIOCDLIFADDR = 0x8118691d - SIOCGDRVSPEC = 0xc01c697b - SIOCGETSGCNT = 0xc0147210 - SIOCGETVIFCNT = 0xc014720f - SIOCGHIWAT = 0x40047301 - SIOCGIFADDR = 0xc0206921 - SIOCGIFBRDADDR = 0xc0206923 - SIOCGIFCAP = 0xc020691f - SIOCGIFCONF = 0xc0086924 - SIOCGIFDESCR = 0xc020692a - SIOCGIFDSTADDR = 0xc0206922 - SIOCGIFFLAGS = 0xc0206911 - SIOCGIFGENERIC = 0xc020693a - SIOCGIFGMEMB = 0xc024698a - SIOCGIFGROUP = 0xc0246988 - SIOCGIFINDEX = 0xc0206920 - SIOCGIFMAC = 0xc0206926 - SIOCGIFMEDIA = 0xc0286938 - SIOCGIFMETRIC = 0xc0206917 - SIOCGIFMTU = 0xc0206933 - SIOCGIFNETMASK = 0xc0206925 - SIOCGIFPDSTADDR = 0xc0206948 - SIOCGIFPHYS = 0xc0206935 - SIOCGIFPSRCADDR = 0xc0206947 - SIOCGIFSTATUS = 0xc331693b - SIOCGLIFADDR = 0xc118691c - SIOCGLIFPHYADDR = 0xc118694b - SIOCGLOWAT = 0x40047303 - SIOCGPGRP = 0x40047309 - SIOCGPRIVATE_0 = 0xc0206950 - SIOCGPRIVATE_1 = 0xc0206951 - SIOCIFCREATE = 0xc020697a - SIOCIFCREATE2 = 0xc020697c - SIOCIFDESTROY = 0x80206979 - SIOCIFGCLONERS = 0xc00c6978 - SIOCSDRVSPEC = 0x801c697b - SIOCSHIWAT = 0x80047300 - SIOCSIFADDR = 0x8020690c - SIOCSIFBRDADDR = 0x80206913 - SIOCSIFCAP = 0x8020691e - SIOCSIFDESCR = 0x80206929 - SIOCSIFDSTADDR = 0x8020690e - SIOCSIFFLAGS = 0x80206910 - SIOCSIFGENERIC = 0x80206939 - SIOCSIFLLADDR = 0x8020693c - SIOCSIFMAC = 0x80206927 - SIOCSIFMEDIA = 0xc0206937 - SIOCSIFMETRIC = 0x80206918 - SIOCSIFMTU = 0x80206934 - SIOCSIFNAME = 0x80206928 - SIOCSIFNETMASK = 0x80206916 - SIOCSIFPHYADDR = 0x80406946 - SIOCSIFPHYS = 0x80206936 - SIOCSIFRVNET = 0xc020695b - SIOCSIFVNET = 0xc020695a - SIOCSLIFPHYADDR = 0x8118694a - SIOCSLOWAT = 0x80047302 - SIOCSPGRP = 0x80047308 - SOCK_DGRAM = 0x2 - SOCK_MAXADDRLEN = 0xff - SOCK_RAW = 0x3 - SOCK_RDM = 0x4 - SOCK_SEQPACKET = 0x5 - SOCK_STREAM = 0x1 - SOL_SOCKET = 0xffff - SOMAXCONN = 0x80 - SO_ACCEPTCONN = 0x2 - SO_ACCEPTFILTER = 0x1000 - SO_BINTIME = 0x2000 - SO_BROADCAST = 0x20 - SO_DEBUG = 0x1 - SO_DONTROUTE = 0x10 - SO_ERROR = 0x1007 - SO_KEEPALIVE = 0x8 - SO_LABEL = 0x1009 - SO_LINGER = 0x80 - SO_LISTENINCQLEN = 0x1013 - SO_LISTENQLEN = 0x1012 - SO_LISTENQLIMIT = 0x1011 - SO_NOSIGPIPE = 0x800 - SO_NO_DDP = 0x8000 - SO_NO_OFFLOAD = 0x4000 - SO_OOBINLINE = 0x100 - SO_PEERLABEL = 0x1010 - SO_RCVBUF = 0x1002 - SO_RCVLOWAT = 0x1004 - SO_RCVTIMEO = 0x1006 - SO_REUSEADDR = 0x4 - SO_REUSEPORT = 0x200 - SO_SETFIB = 0x1014 - SO_SNDBUF = 0x1001 - SO_SNDLOWAT = 0x1003 - SO_SNDTIMEO = 0x1005 - SO_TIMESTAMP = 0x400 - SO_TYPE = 0x1008 - SO_USELOOPBACK = 0x40 - TCP_CA_NAME_MAX = 0x10 - TCP_CONGESTION = 0x40 - TCP_INFO = 0x20 - TCP_MAXBURST = 0x4 - TCP_MAXHLEN = 0x3c - TCP_MAXOLEN = 0x28 - TCP_MAXSEG = 0x2 - TCP_MAXWIN = 0xffff - TCP_MAX_SACK = 0x4 - TCP_MAX_WINSHIFT = 0xe - TCP_MD5SIG = 0x10 - TCP_MINMSS = 0xd8 - TCP_MSS = 0x200 - TCP_NODELAY = 0x1 - TCP_NOOPT = 0x8 - TCP_NOPUSH = 0x4 - WCONTINUED = 0x4 - WCOREFLAG = 0x80 - WLINUXCLONE = 0x80000000 - WNOHANG = 0x1 - WNOWAIT = 0x8 - WSTOPPED = 0x2 - WUNTRACED = 0x2 + AF_APPLETALK = 0x10 + AF_ARP = 0x23 + AF_ATM = 0x1e + AF_BLUETOOTH = 0x24 + AF_CCITT = 0xa + AF_CHAOS = 0x5 + AF_CNT = 0x15 + AF_COIP = 0x14 + AF_DATAKIT = 0x9 + AF_DECnet = 0xc + AF_DLI = 0xd + AF_E164 = 0x1a + AF_ECMA = 0x8 + AF_HYLINK = 0xf + AF_IEEE80211 = 0x25 + AF_IMPLINK = 0x3 + AF_INET = 0x2 + AF_INET6 = 0x1c + AF_IPX = 0x17 + AF_ISDN = 0x1a + AF_ISO = 0x7 + AF_LAT = 0xe + AF_LINK = 0x12 + AF_LOCAL = 0x1 + AF_MAX = 0x26 + AF_NATM = 0x1d + AF_NETBIOS = 0x6 + AF_NETGRAPH = 0x20 + AF_OSI = 0x7 + AF_PUP = 0x4 + AF_ROUTE = 0x11 + AF_SCLUSTER = 0x22 + AF_SIP = 0x18 + AF_SLOW = 0x21 + AF_SNA = 0xb + AF_UNIX = 0x1 + AF_UNSPEC = 0 + AF_VENDOR00 = 0x27 + AF_VENDOR01 = 0x29 + AF_VENDOR02 = 0x2b + AF_VENDOR03 = 0x2d + AF_VENDOR04 = 0x2f + AF_VENDOR05 = 0x31 + AF_VENDOR06 = 0x33 + AF_VENDOR07 = 0x35 + AF_VENDOR08 = 0x37 + AF_VENDOR09 = 0x39 + AF_VENDOR10 = 0x3b + AF_VENDOR11 = 0x3d + AF_VENDOR12 = 0x3f + AF_VENDOR13 = 0x41 + AF_VENDOR14 = 0x43 + AF_VENDOR15 = 0x45 + AF_VENDOR16 = 0x47 + AF_VENDOR17 = 0x49 + AF_VENDOR18 = 0x4b + AF_VENDOR19 = 0x4d + AF_VENDOR20 = 0x4f + AF_VENDOR21 = 0x51 + AF_VENDOR22 = 0x53 + AF_VENDOR23 = 0x55 + AF_VENDOR24 = 0x57 + AF_VENDOR25 = 0x59 + AF_VENDOR26 = 0x5b + AF_VENDOR27 = 0x5d + AF_VENDOR28 = 0x5f + AF_VENDOR29 = 0x61 + AF_VENDOR30 = 0x63 + AF_VENDOR31 = 0x65 + AF_VENDOR32 = 0x67 + AF_VENDOR33 = 0x69 + AF_VENDOR34 = 0x6b + AF_VENDOR35 = 0x6d + AF_VENDOR36 = 0x6f + AF_VENDOR37 = 0x71 + AF_VENDOR38 = 0x73 + AF_VENDOR39 = 0x75 + AF_VENDOR40 = 0x77 + AF_VENDOR41 = 0x79 + AF_VENDOR42 = 0x7b + AF_VENDOR43 = 0x7d + AF_VENDOR44 = 0x7f + AF_VENDOR45 = 0x81 + AF_VENDOR46 = 0x83 + AF_VENDOR47 = 0x85 + BIOCFEEDBACK = 0x8004427c + BIOCFLUSH = 0x20004268 + BIOCGBLEN = 0x40044266 + BIOCGDIRECTION = 0x40044276 + BIOCGDLT = 0x4004426a + BIOCGDLTLIST = 0xc0084279 + BIOCGETBUFMODE = 0x4004427d + BIOCGETIF = 0x4020426b + BIOCGETZMAX = 0x4004427f + BIOCGHDRCMPLT = 0x40044274 + BIOCGRSIG = 0x40044272 + BIOCGRTIMEOUT = 0x4008426e + BIOCGSEESENT = 0x40044276 + BIOCGSTATS = 0x4008426f + BIOCIMMEDIATE = 0x80044270 + BIOCLOCK = 0x2000427a + BIOCPROMISC = 0x20004269 + BIOCROTZBUF = 0x400c4280 + BIOCSBLEN = 0xc0044266 + BIOCSDIRECTION = 0x80044277 + BIOCSDLT = 0x80044278 + BIOCSETBUFMODE = 0x8004427e + BIOCSETF = 0x80084267 + BIOCSETFNR = 0x80084282 + BIOCSETIF = 0x8020426c + BIOCSETWF = 0x8008427b + BIOCSETZBUF = 0x800c4281 + BIOCSHDRCMPLT = 0x80044275 + BIOCSRSIG = 0x80044273 + BIOCSRTIMEOUT = 0x8008426d + BIOCSSEESENT = 0x80044277 + BIOCVERSION = 0x40044271 + BPF_A = 0x10 + BPF_ABS = 0x20 + BPF_ADD = 0 + BPF_ALIGNMENT = 0x4 + BPF_ALU = 0x4 + BPF_AND = 0x50 + BPF_B = 0x10 + BPF_BUFMODE_BUFFER = 0x1 + BPF_BUFMODE_ZBUF = 0x2 + BPF_DIV = 0x30 + BPF_H = 0x8 + BPF_IMM = 0 + BPF_IND = 0x40 + BPF_JA = 0 + BPF_JEQ = 0x10 + BPF_JGE = 0x30 + BPF_JGT = 0x20 + BPF_JMP = 0x5 + BPF_JSET = 0x40 + BPF_K = 0 + BPF_LD = 0 + BPF_LDX = 0x1 + BPF_LEN = 0x80 + BPF_LSH = 0x60 + BPF_MAJOR_VERSION = 0x1 + BPF_MAXBUFSIZE = 0x80000 + BPF_MAXINSNS = 0x200 + BPF_MEM = 0x60 + BPF_MEMWORDS = 0x10 + BPF_MINBUFSIZE = 0x20 + BPF_MINOR_VERSION = 0x1 + BPF_MISC = 0x7 + BPF_MSH = 0xa0 + BPF_MUL = 0x20 + BPF_NEG = 0x80 + BPF_OR = 0x40 + BPF_RELEASE = 0x30bb6 + BPF_RET = 0x6 + BPF_RSH = 0x70 + BPF_ST = 0x2 + BPF_STX = 0x3 + BPF_SUB = 0x10 + BPF_TAX = 0 + BPF_TXA = 0x80 + BPF_W = 0 + BPF_X = 0x8 + CTL_MAXNAME = 0x18 + CTL_NET = 0x4 + DLT_A429 = 0xb8 + DLT_A653_ICM = 0xb9 + DLT_AIRONET_HEADER = 0x78 + DLT_APPLE_IP_OVER_IEEE1394 = 0x8a + DLT_ARCNET = 0x7 + DLT_ARCNET_LINUX = 0x81 + DLT_ATM_CLIP = 0x13 + DLT_ATM_RFC1483 = 0xb + DLT_AURORA = 0x7e + DLT_AX25 = 0x3 + DLT_AX25_KISS = 0xca + DLT_BACNET_MS_TP = 0xa5 + DLT_BLUETOOTH_HCI_H4 = 0xbb + DLT_BLUETOOTH_HCI_H4_WITH_PHDR = 0xc9 + DLT_CAN20B = 0xbe + DLT_CHAOS = 0x5 + DLT_CHDLC = 0x68 + DLT_CISCO_IOS = 0x76 + DLT_C_HDLC = 0x68 + DLT_C_HDLC_WITH_DIR = 0xcd + DLT_DOCSIS = 0x8f + DLT_ECONET = 0x73 + DLT_EN10MB = 0x1 + DLT_EN3MB = 0x2 + DLT_ENC = 0x6d + DLT_ERF = 0xc5 + DLT_ERF_ETH = 0xaf + DLT_ERF_POS = 0xb0 + DLT_FDDI = 0xa + DLT_FLEXRAY = 0xd2 + DLT_FRELAY = 0x6b + DLT_FRELAY_WITH_DIR = 0xce + DLT_GCOM_SERIAL = 0xad + DLT_GCOM_T1E1 = 0xac + DLT_GPF_F = 0xab + DLT_GPF_T = 0xaa + DLT_GPRS_LLC = 0xa9 + DLT_HHDLC = 0x79 + DLT_IBM_SN = 0x92 + DLT_IBM_SP = 0x91 + DLT_IEEE802 = 0x6 + DLT_IEEE802_11 = 0x69 + DLT_IEEE802_11_RADIO = 0x7f + DLT_IEEE802_11_RADIO_AVS = 0xa3 + DLT_IEEE802_15_4 = 0xc3 + DLT_IEEE802_15_4_LINUX = 0xbf + DLT_IEEE802_15_4_NONASK_PHY = 0xd7 + DLT_IEEE802_16_MAC_CPS = 0xbc + DLT_IEEE802_16_MAC_CPS_RADIO = 0xc1 + DLT_IPFILTER = 0x74 + DLT_IPMB = 0xc7 + DLT_IPMB_LINUX = 0xd1 + DLT_IP_OVER_FC = 0x7a + DLT_JUNIPER_ATM1 = 0x89 + DLT_JUNIPER_ATM2 = 0x87 + DLT_JUNIPER_CHDLC = 0xb5 + DLT_JUNIPER_ES = 0x84 + DLT_JUNIPER_ETHER = 0xb2 + DLT_JUNIPER_FRELAY = 0xb4 + DLT_JUNIPER_GGSN = 0x85 + DLT_JUNIPER_ISM = 0xc2 + DLT_JUNIPER_MFR = 0x86 + DLT_JUNIPER_MLFR = 0x83 + DLT_JUNIPER_MLPPP = 0x82 + DLT_JUNIPER_MONITOR = 0xa4 + DLT_JUNIPER_PIC_PEER = 0xae + DLT_JUNIPER_PPP = 0xb3 + DLT_JUNIPER_PPPOE = 0xa7 + DLT_JUNIPER_PPPOE_ATM = 0xa8 + DLT_JUNIPER_SERVICES = 0x88 + DLT_JUNIPER_ST = 0xc8 + DLT_JUNIPER_VP = 0xb7 + DLT_LAPB_WITH_DIR = 0xcf + DLT_LAPD = 0xcb + DLT_LIN = 0xd4 + DLT_LINUX_IRDA = 0x90 + DLT_LINUX_LAPD = 0xb1 + DLT_LINUX_PPP_WITHDIRECTION = 0xa6 + DLT_LINUX_SLL = 0x71 + DLT_LOOP = 0x6c + DLT_LTALK = 0x72 + DLT_MFR = 0xb6 + DLT_MOST = 0xd3 + DLT_MTP2 = 0x8c + DLT_MTP2_WITH_PHDR = 0x8b + DLT_MTP3 = 0x8d + DLT_NULL = 0 + DLT_PCI_EXP = 0x7d + DLT_PFLOG = 0x75 + DLT_PFSYNC = 0x79 + DLT_PPI = 0xc0 + DLT_PPP = 0x9 + DLT_PPP_BSDOS = 0x10 + DLT_PPP_ETHER = 0x33 + DLT_PPP_PPPD = 0xa6 + DLT_PPP_SERIAL = 0x32 + DLT_PPP_WITH_DIR = 0xcc + DLT_PPP_WITH_DIRECTION = 0xa6 + DLT_PRISM_HEADER = 0x77 + DLT_PRONET = 0x4 + DLT_RAIF1 = 0xc6 + DLT_RAW = 0xc + DLT_RIO = 0x7c + DLT_SCCP = 0x8e + DLT_SITA = 0xc4 + DLT_SLIP = 0x8 + DLT_SLIP_BSDOS = 0xf + DLT_SUNATM = 0x7b + DLT_SYMANTEC_FIREWALL = 0x63 + DLT_TZSP = 0x80 + DLT_USB = 0xba + DLT_USB_LINUX = 0xbd + DLT_USER0 = 0x93 + DLT_USER1 = 0x94 + DLT_USER10 = 0x9d + DLT_USER11 = 0x9e + DLT_USER12 = 0x9f + DLT_USER13 = 0xa0 + DLT_USER14 = 0xa1 + DLT_USER15 = 0xa2 + DLT_USER2 = 0x95 + DLT_USER3 = 0x96 + DLT_USER4 = 0x97 + DLT_USER5 = 0x98 + DLT_USER6 = 0x99 + DLT_USER7 = 0x9a + DLT_USER8 = 0x9b + DLT_USER9 = 0x9c + DLT_X2E_SERIAL = 0xd5 + DLT_X2E_XORAYA = 0xd6 + DT_BLK = 0x6 + DT_CHR = 0x2 + DT_DIR = 0x4 + DT_FIFO = 0x1 + DT_LNK = 0xa + DT_REG = 0x8 + DT_SOCK = 0xc + DT_UNKNOWN = 0 + DT_WHT = 0xe + E2BIG = 0x7 + EACCES = 0xd + EADDRINUSE = 0x30 + EADDRNOTAVAIL = 0x31 + EAFNOSUPPORT = 0x2f + EAGAIN = 0x23 + EALREADY = 0x25 + EAUTH = 0x50 + EBADF = 0x9 + EBADMSG = 0x59 + EBADRPC = 0x48 + EBUSY = 0x10 + ECANCELED = 0x55 + ECHILD = 0xa + ECONNABORTED = 0x35 + ECONNREFUSED = 0x3d + ECONNRESET = 0x36 + EDEADLK = 0xb + EDESTADDRREQ = 0x27 + EDOM = 0x21 + EDOOFUS = 0x58 + EDQUOT = 0x45 + EEXIST = 0x11 + EFAULT = 0xe + EFBIG = 0x1b + EFTYPE = 0x4f + EHOSTDOWN = 0x40 + EHOSTUNREACH = 0x41 + EIDRM = 0x52 + EILSEQ = 0x56 + EINPROGRESS = 0x24 + EINTR = 0x4 + EINVAL = 0x16 + EIO = 0x5 + EISCONN = 0x38 + EISDIR = 0x15 + ELAST = 0x5d + ELOOP = 0x3e + EMFILE = 0x18 + EMLINK = 0x1f + EMSGSIZE = 0x28 + EMULTIHOP = 0x5a + ENAMETOOLONG = 0x3f + ENEEDAUTH = 0x51 + ENETDOWN = 0x32 + ENETRESET = 0x34 + ENETUNREACH = 0x33 + ENFILE = 0x17 + ENOATTR = 0x57 + ENOBUFS = 0x37 + ENODEV = 0x13 + ENOENT = 0x2 + ENOEXEC = 0x8 + ENOLCK = 0x4d + ENOLINK = 0x5b + ENOMEM = 0xc + ENOMSG = 0x53 + ENOPROTOOPT = 0x2a + ENOSPC = 0x1c + ENOSYS = 0x4e + ENOTBLK = 0xf + ENOTCAPABLE = 0x5d + ENOTCONN = 0x39 + ENOTDIR = 0x14 + ENOTEMPTY = 0x42 + ENOTSOCK = 0x26 + ENOTSUP = 0x2d + ENOTTY = 0x19 + ENXIO = 0x6 + EOPNOTSUPP = 0x2d + EOVERFLOW = 0x54 + EPERM = 0x1 + EPFNOSUPPORT = 0x2e + EPIPE = 0x20 + EPROCLIM = 0x43 + EPROCUNAVAIL = 0x4c + EPROGMISMATCH = 0x4b + EPROGUNAVAIL = 0x4a + EPROTO = 0x5c + EPROTONOSUPPORT = 0x2b + EPROTOTYPE = 0x29 + ERANGE = 0x22 + EREMOTE = 0x47 + EROFS = 0x1e + ERPCMISMATCH = 0x49 + ESHUTDOWN = 0x3a + ESOCKTNOSUPPORT = 0x2c + ESPIPE = 0x1d + ESRCH = 0x3 + ESTALE = 0x46 + ETIMEDOUT = 0x3c + ETOOMANYREFS = 0x3b + ETXTBSY = 0x1a + EUSERS = 0x44 + EVFILT_AIO = -0x3 + EVFILT_FS = -0x9 + EVFILT_LIO = -0xa + EVFILT_PROC = -0x5 + EVFILT_READ = -0x1 + EVFILT_SIGNAL = -0x6 + EVFILT_SYSCOUNT = 0xb + EVFILT_TIMER = -0x7 + EVFILT_USER = -0xb + EVFILT_VNODE = -0x4 + EVFILT_WRITE = -0x2 + EV_ADD = 0x1 + EV_CLEAR = 0x20 + EV_DELETE = 0x2 + EV_DISABLE = 0x8 + EV_DISPATCH = 0x80 + EV_ENABLE = 0x4 + EV_EOF = 0x8000 + EV_ERROR = 0x4000 + EV_FLAG1 = 0x2000 + EV_ONESHOT = 0x10 + EV_RECEIPT = 0x40 + EV_SYSFLAGS = 0xf000 + EWOULDBLOCK = 0x23 + EXDEV = 0x12 + FD_CLOEXEC = 0x1 + FD_SETSIZE = 0x400 + F_CANCEL = 0x5 + F_DUP2FD = 0xa + F_DUPFD = 0 + F_GETFD = 0x1 + F_GETFL = 0x3 + F_GETLK = 0xb + F_GETOWN = 0x5 + F_OGETLK = 0x7 + F_OSETLK = 0x8 + F_OSETLKW = 0x9 + F_RDAHEAD = 0x10 + F_RDLCK = 0x1 + F_READAHEAD = 0xf + F_SETFD = 0x2 + F_SETFL = 0x4 + F_SETLK = 0xc + F_SETLKW = 0xd + F_SETLK_REMOTE = 0xe + F_SETOWN = 0x6 + F_UNLCK = 0x2 + F_UNLCKSYS = 0x4 + F_WRLCK = 0x3 + IFF_ALLMULTI = 0x200 + IFF_ALTPHYS = 0x4000 + IFF_BROADCAST = 0x2 + IFF_CANTCHANGE = 0x208f72 + IFF_DEBUG = 0x4 + IFF_DRV_OACTIVE = 0x400 + IFF_DRV_RUNNING = 0x40 + IFF_DYING = 0x200000 + IFF_LINK0 = 0x1000 + IFF_LINK1 = 0x2000 + IFF_LINK2 = 0x4000 + IFF_LOOPBACK = 0x8 + IFF_MONITOR = 0x40000 + IFF_MULTICAST = 0x8000 + IFF_NOARP = 0x80 + IFF_OACTIVE = 0x400 + IFF_POINTOPOINT = 0x10 + IFF_PPROMISC = 0x20000 + IFF_PROMISC = 0x100 + IFF_RENAMING = 0x400000 + IFF_RUNNING = 0x40 + IFF_SIMPLEX = 0x800 + IFF_SMART = 0x20 + IFF_STATICARP = 0x80000 + IFF_UP = 0x1 + IFNAMSIZ = 0x10 + IFT_1822 = 0x2 + IFT_A12MPPSWITCH = 0x82 + IFT_AAL2 = 0xbb + IFT_AAL5 = 0x31 + IFT_ADSL = 0x5e + IFT_AFLANE8023 = 0x3b + IFT_AFLANE8025 = 0x3c + IFT_ARAP = 0x58 + IFT_ARCNET = 0x23 + IFT_ARCNETPLUS = 0x24 + IFT_ASYNC = 0x54 + IFT_ATM = 0x25 + IFT_ATMDXI = 0x69 + IFT_ATMFUNI = 0x6a + IFT_ATMIMA = 0x6b + IFT_ATMLOGICAL = 0x50 + IFT_ATMRADIO = 0xbd + IFT_ATMSUBINTERFACE = 0x86 + IFT_ATMVCIENDPT = 0xc2 + IFT_ATMVIRTUAL = 0x95 + IFT_BGPPOLICYACCOUNTING = 0xa2 + IFT_BRIDGE = 0xd1 + IFT_BSC = 0x53 + IFT_CARP = 0xf8 + IFT_CCTEMUL = 0x3d + IFT_CEPT = 0x13 + IFT_CES = 0x85 + IFT_CHANNEL = 0x46 + IFT_CNR = 0x55 + IFT_COFFEE = 0x84 + IFT_COMPOSITELINK = 0x9b + IFT_DCN = 0x8d + IFT_DIGITALPOWERLINE = 0x8a + IFT_DIGITALWRAPPEROVERHEADCHANNEL = 0xba + IFT_DLSW = 0x4a + IFT_DOCSCABLEDOWNSTREAM = 0x80 + IFT_DOCSCABLEMACLAYER = 0x7f + IFT_DOCSCABLEUPSTREAM = 0x81 + IFT_DS0 = 0x51 + IFT_DS0BUNDLE = 0x52 + IFT_DS1FDL = 0xaa + IFT_DS3 = 0x1e + IFT_DTM = 0x8c + IFT_DVBASILN = 0xac + IFT_DVBASIOUT = 0xad + IFT_DVBRCCDOWNSTREAM = 0x93 + IFT_DVBRCCMACLAYER = 0x92 + IFT_DVBRCCUPSTREAM = 0x94 + IFT_ENC = 0xf4 + IFT_EON = 0x19 + IFT_EPLRS = 0x57 + IFT_ESCON = 0x49 + IFT_ETHER = 0x6 + IFT_FAITH = 0xf2 + IFT_FAST = 0x7d + IFT_FASTETHER = 0x3e + IFT_FASTETHERFX = 0x45 + IFT_FDDI = 0xf + IFT_FIBRECHANNEL = 0x38 + IFT_FRAMERELAYINTERCONNECT = 0x3a + IFT_FRAMERELAYMPI = 0x5c + IFT_FRDLCIENDPT = 0xc1 + IFT_FRELAY = 0x20 + IFT_FRELAYDCE = 0x2c + IFT_FRF16MFRBUNDLE = 0xa3 + IFT_FRFORWARD = 0x9e + IFT_G703AT2MB = 0x43 + IFT_G703AT64K = 0x42 + IFT_GIF = 0xf0 + IFT_GIGABITETHERNET = 0x75 + IFT_GR303IDT = 0xb2 + IFT_GR303RDT = 0xb1 + IFT_H323GATEKEEPER = 0xa4 + IFT_H323PROXY = 0xa5 + IFT_HDH1822 = 0x3 + IFT_HDLC = 0x76 + IFT_HDSL2 = 0xa8 + IFT_HIPERLAN2 = 0xb7 + IFT_HIPPI = 0x2f + IFT_HIPPIINTERFACE = 0x39 + IFT_HOSTPAD = 0x5a + IFT_HSSI = 0x2e + IFT_HY = 0xe + IFT_IBM370PARCHAN = 0x48 + IFT_IDSL = 0x9a + IFT_IEEE1394 = 0x90 + IFT_IEEE80211 = 0x47 + IFT_IEEE80212 = 0x37 + IFT_IEEE8023ADLAG = 0xa1 + IFT_IFGSN = 0x91 + IFT_IMT = 0xbe + IFT_INTERLEAVE = 0x7c + IFT_IP = 0x7e + IFT_IPFORWARD = 0x8e + IFT_IPOVERATM = 0x72 + IFT_IPOVERCDLC = 0x6d + IFT_IPOVERCLAW = 0x6e + IFT_IPSWITCH = 0x4e + IFT_IPXIP = 0xf9 + IFT_ISDN = 0x3f + IFT_ISDNBASIC = 0x14 + IFT_ISDNPRIMARY = 0x15 + IFT_ISDNS = 0x4b + IFT_ISDNU = 0x4c + IFT_ISO88022LLC = 0x29 + IFT_ISO88023 = 0x7 + IFT_ISO88024 = 0x8 + IFT_ISO88025 = 0x9 + IFT_ISO88025CRFPINT = 0x62 + IFT_ISO88025DTR = 0x56 + IFT_ISO88025FIBER = 0x73 + IFT_ISO88026 = 0xa + IFT_ISUP = 0xb3 + IFT_L2VLAN = 0x87 + IFT_L3IPVLAN = 0x88 + IFT_L3IPXVLAN = 0x89 + IFT_LAPB = 0x10 + IFT_LAPD = 0x4d + IFT_LAPF = 0x77 + IFT_LOCALTALK = 0x2a + IFT_LOOP = 0x18 + IFT_MEDIAMAILOVERIP = 0x8b + IFT_MFSIGLINK = 0xa7 + IFT_MIOX25 = 0x26 + IFT_MODEM = 0x30 + IFT_MPC = 0x71 + IFT_MPLS = 0xa6 + IFT_MPLSTUNNEL = 0x96 + IFT_MSDSL = 0x8f + IFT_MVL = 0xbf + IFT_MYRINET = 0x63 + IFT_NFAS = 0xaf + IFT_NSIP = 0x1b + IFT_OPTICALCHANNEL = 0xc3 + IFT_OPTICALTRANSPORT = 0xc4 + IFT_OTHER = 0x1 + IFT_P10 = 0xc + IFT_P80 = 0xd + IFT_PARA = 0x22 + IFT_PFLOG = 0xf6 + IFT_PFSYNC = 0xf7 + IFT_PLC = 0xae + IFT_POS = 0xab + IFT_PPP = 0x17 + IFT_PPPMULTILINKBUNDLE = 0x6c + IFT_PROPBWAP2MP = 0xb8 + IFT_PROPCNLS = 0x59 + IFT_PROPDOCSWIRELESSDOWNSTREAM = 0xb5 + IFT_PROPDOCSWIRELESSMACLAYER = 0xb4 + IFT_PROPDOCSWIRELESSUPSTREAM = 0xb6 + IFT_PROPMUX = 0x36 + IFT_PROPVIRTUAL = 0x35 + IFT_PROPWIRELESSP2P = 0x9d + IFT_PTPSERIAL = 0x16 + IFT_PVC = 0xf1 + IFT_QLLC = 0x44 + IFT_RADIOMAC = 0xbc + IFT_RADSL = 0x5f + IFT_REACHDSL = 0xc0 + IFT_RFC1483 = 0x9f + IFT_RS232 = 0x21 + IFT_RSRB = 0x4f + IFT_SDLC = 0x11 + IFT_SDSL = 0x60 + IFT_SHDSL = 0xa9 + IFT_SIP = 0x1f + IFT_SLIP = 0x1c + IFT_SMDSDXI = 0x2b + IFT_SMDSICIP = 0x34 + IFT_SONET = 0x27 + IFT_SONETOVERHEADCHANNEL = 0xb9 + IFT_SONETPATH = 0x32 + IFT_SONETVT = 0x33 + IFT_SRP = 0x97 + IFT_SS7SIGLINK = 0x9c + IFT_STACKTOSTACK = 0x6f + IFT_STARLAN = 0xb + IFT_STF = 0xd7 + IFT_T1 = 0x12 + IFT_TDLC = 0x74 + IFT_TERMPAD = 0x5b + IFT_TR008 = 0xb0 + IFT_TRANSPHDLC = 0x7b + IFT_TUNNEL = 0x83 + IFT_ULTRA = 0x1d + IFT_USB = 0xa0 + IFT_V11 = 0x40 + IFT_V35 = 0x2d + IFT_V36 = 0x41 + IFT_V37 = 0x78 + IFT_VDSL = 0x61 + IFT_VIRTUALIPADDRESS = 0x70 + IFT_VOICEEM = 0x64 + IFT_VOICEENCAP = 0x67 + IFT_VOICEFXO = 0x65 + IFT_VOICEFXS = 0x66 + IFT_VOICEOVERATM = 0x98 + IFT_VOICEOVERFRAMERELAY = 0x99 + IFT_VOICEOVERIP = 0x68 + IFT_X213 = 0x5d + IFT_X25 = 0x5 + IFT_X25DDN = 0x4 + IFT_X25HUNTGROUP = 0x7a + IFT_X25MLP = 0x79 + IFT_X25PLE = 0x28 + IFT_XETHER = 0x1a + IN_CLASSA_HOST = 0xffffff + IN_CLASSA_MAX = 0x80 + IN_CLASSA_NET = 0xff000000 + IN_CLASSA_NSHIFT = 0x18 + IN_CLASSB_HOST = 0xffff + IN_CLASSB_MAX = 0x10000 + IN_CLASSB_NET = 0xffff0000 + IN_CLASSB_NSHIFT = 0x10 + IN_CLASSC_HOST = 0xff + IN_CLASSC_NET = 0xffffff00 + IN_CLASSC_NSHIFT = 0x8 + IN_CLASSD_HOST = 0xfffffff + IN_CLASSD_NET = 0xf0000000 + IN_CLASSD_NSHIFT = 0x1c + IN_LOOPBACKNET = 0x7f + IPPROTO_3PC = 0x22 + IPPROTO_ADFS = 0x44 + IPPROTO_AH = 0x33 + IPPROTO_AHIP = 0x3d + IPPROTO_APES = 0x63 + IPPROTO_ARGUS = 0xd + IPPROTO_AX25 = 0x5d + IPPROTO_BHA = 0x31 + IPPROTO_BLT = 0x1e + IPPROTO_BRSATMON = 0x4c + IPPROTO_CARP = 0x70 + IPPROTO_CFTP = 0x3e + IPPROTO_CHAOS = 0x10 + IPPROTO_CMTP = 0x26 + IPPROTO_CPHB = 0x49 + IPPROTO_CPNX = 0x48 + IPPROTO_DDP = 0x25 + IPPROTO_DGP = 0x56 + IPPROTO_DIVERT = 0x102 + IPPROTO_DONE = 0x101 + IPPROTO_DSTOPTS = 0x3c + IPPROTO_EGP = 0x8 + IPPROTO_EMCON = 0xe + IPPROTO_ENCAP = 0x62 + IPPROTO_EON = 0x50 + IPPROTO_ESP = 0x32 + IPPROTO_ETHERIP = 0x61 + IPPROTO_FRAGMENT = 0x2c + IPPROTO_GGP = 0x3 + IPPROTO_GMTP = 0x64 + IPPROTO_GRE = 0x2f + IPPROTO_HELLO = 0x3f + IPPROTO_HMP = 0x14 + IPPROTO_HOPOPTS = 0 + IPPROTO_ICMP = 0x1 + IPPROTO_ICMPV6 = 0x3a + IPPROTO_IDP = 0x16 + IPPROTO_IDPR = 0x23 + IPPROTO_IDRP = 0x2d + IPPROTO_IGMP = 0x2 + IPPROTO_IGP = 0x55 + IPPROTO_IGRP = 0x58 + IPPROTO_IL = 0x28 + IPPROTO_INLSP = 0x34 + IPPROTO_INP = 0x20 + IPPROTO_IP = 0 + IPPROTO_IPCOMP = 0x6c + IPPROTO_IPCV = 0x47 + IPPROTO_IPEIP = 0x5e + IPPROTO_IPIP = 0x4 + IPPROTO_IPPC = 0x43 + IPPROTO_IPV4 = 0x4 + IPPROTO_IPV6 = 0x29 + IPPROTO_IRTP = 0x1c + IPPROTO_KRYPTOLAN = 0x41 + IPPROTO_LARP = 0x5b + IPPROTO_LEAF1 = 0x19 + IPPROTO_LEAF2 = 0x1a + IPPROTO_MAX = 0x100 + IPPROTO_MAXID = 0x34 + IPPROTO_MEAS = 0x13 + IPPROTO_MHRP = 0x30 + IPPROTO_MICP = 0x5f + IPPROTO_MOBILE = 0x37 + IPPROTO_MTP = 0x5c + IPPROTO_MUX = 0x12 + IPPROTO_ND = 0x4d + IPPROTO_NHRP = 0x36 + IPPROTO_NONE = 0x3b + IPPROTO_NSP = 0x1f + IPPROTO_NVPII = 0xb + IPPROTO_OLD_DIVERT = 0xfe + IPPROTO_OSPFIGP = 0x59 + IPPROTO_PFSYNC = 0xf0 + IPPROTO_PGM = 0x71 + IPPROTO_PIGP = 0x9 + IPPROTO_PIM = 0x67 + IPPROTO_PRM = 0x15 + IPPROTO_PUP = 0xc + IPPROTO_PVP = 0x4b + IPPROTO_RAW = 0xff + IPPROTO_RCCMON = 0xa + IPPROTO_RDP = 0x1b + IPPROTO_ROUTING = 0x2b + IPPROTO_RSVP = 0x2e + IPPROTO_RVD = 0x42 + IPPROTO_SATEXPAK = 0x40 + IPPROTO_SATMON = 0x45 + IPPROTO_SCCSP = 0x60 + IPPROTO_SCTP = 0x84 + IPPROTO_SDRP = 0x2a + IPPROTO_SEP = 0x21 + IPPROTO_SKIP = 0x39 + IPPROTO_SPACER = 0x7fff + IPPROTO_SRPC = 0x5a + IPPROTO_ST = 0x7 + IPPROTO_SVMTP = 0x52 + IPPROTO_SWIPE = 0x35 + IPPROTO_TCF = 0x57 + IPPROTO_TCP = 0x6 + IPPROTO_TLSP = 0x38 + IPPROTO_TP = 0x1d + IPPROTO_TPXX = 0x27 + IPPROTO_TRUNK1 = 0x17 + IPPROTO_TRUNK2 = 0x18 + IPPROTO_TTP = 0x54 + IPPROTO_UDP = 0x11 + IPPROTO_VINES = 0x53 + IPPROTO_VISA = 0x46 + IPPROTO_VMTP = 0x51 + IPPROTO_WBEXPAK = 0x4f + IPPROTO_WBMON = 0x4e + IPPROTO_WSN = 0x4a + IPPROTO_XNET = 0xf + IPPROTO_XTP = 0x24 + IPV6_AUTOFLOWLABEL = 0x3b + IPV6_BINDANY = 0x40 + IPV6_BINDV6ONLY = 0x1b + IPV6_CHECKSUM = 0x1a + IPV6_DEFAULT_MULTICAST_HOPS = 0x1 + IPV6_DEFAULT_MULTICAST_LOOP = 0x1 + IPV6_DEFHLIM = 0x40 + IPV6_DONTFRAG = 0x3e + IPV6_DSTOPTS = 0x32 + IPV6_FAITH = 0x1d + IPV6_FLOWINFO_MASK = 0xffffff0f + IPV6_FLOWLABEL_MASK = 0xffff0f00 + IPV6_FRAGTTL = 0x78 + IPV6_FW_ADD = 0x1e + IPV6_FW_DEL = 0x1f + IPV6_FW_FLUSH = 0x20 + IPV6_FW_GET = 0x22 + IPV6_FW_ZERO = 0x21 + IPV6_HLIMDEC = 0x1 + IPV6_HOPLIMIT = 0x2f + IPV6_HOPOPTS = 0x31 + IPV6_IPSEC_POLICY = 0x1c + IPV6_JOIN_GROUP = 0xc + IPV6_LEAVE_GROUP = 0xd + IPV6_MAXHLIM = 0xff + IPV6_MAXOPTHDR = 0x800 + IPV6_MAXPACKET = 0xffff + IPV6_MAX_GROUP_SRC_FILTER = 0x200 + IPV6_MAX_MEMBERSHIPS = 0xfff + IPV6_MAX_SOCK_SRC_FILTER = 0x80 + IPV6_MIN_MEMBERSHIPS = 0x1f + IPV6_MMTU = 0x500 + IPV6_MSFILTER = 0x4a + IPV6_MULTICAST_HOPS = 0xa + IPV6_MULTICAST_IF = 0x9 + IPV6_MULTICAST_LOOP = 0xb + IPV6_NEXTHOP = 0x30 + IPV6_PATHMTU = 0x2c + IPV6_PKTINFO = 0x2e + IPV6_PORTRANGE = 0xe + IPV6_PORTRANGE_DEFAULT = 0 + IPV6_PORTRANGE_HIGH = 0x1 + IPV6_PORTRANGE_LOW = 0x2 + IPV6_PREFER_TEMPADDR = 0x3f + IPV6_RECVDSTOPTS = 0x28 + IPV6_RECVHOPLIMIT = 0x25 + IPV6_RECVHOPOPTS = 0x27 + IPV6_RECVPATHMTU = 0x2b + IPV6_RECVPKTINFO = 0x24 + IPV6_RECVRTHDR = 0x26 + IPV6_RECVTCLASS = 0x39 + IPV6_RTHDR = 0x33 + IPV6_RTHDRDSTOPTS = 0x23 + IPV6_RTHDR_LOOSE = 0 + IPV6_RTHDR_STRICT = 0x1 + IPV6_RTHDR_TYPE_0 = 0 + IPV6_SOCKOPT_RESERVED1 = 0x3 + IPV6_TCLASS = 0x3d + IPV6_UNICAST_HOPS = 0x4 + IPV6_USE_MIN_MTU = 0x2a + IPV6_V6ONLY = 0x1b + IPV6_VERSION = 0x60 + IPV6_VERSION_MASK = 0xf0 + IP_ADD_MEMBERSHIP = 0xc + IP_ADD_SOURCE_MEMBERSHIP = 0x46 + IP_BINDANY = 0x18 + IP_BLOCK_SOURCE = 0x48 + IP_DEFAULT_MULTICAST_LOOP = 0x1 + IP_DEFAULT_MULTICAST_TTL = 0x1 + IP_DF = 0x4000 + IP_DONTFRAG = 0x43 + IP_DROP_MEMBERSHIP = 0xd + IP_DROP_SOURCE_MEMBERSHIP = 0x47 + IP_DUMMYNET3 = 0x31 + IP_DUMMYNET_CONFIGURE = 0x3c + IP_DUMMYNET_DEL = 0x3d + IP_DUMMYNET_FLUSH = 0x3e + IP_DUMMYNET_GET = 0x40 + IP_FAITH = 0x16 + IP_FW3 = 0x30 + IP_FW_ADD = 0x32 + IP_FW_DEL = 0x33 + IP_FW_FLUSH = 0x34 + IP_FW_GET = 0x36 + IP_FW_NAT_CFG = 0x38 + IP_FW_NAT_DEL = 0x39 + IP_FW_NAT_GET_CONFIG = 0x3a + IP_FW_NAT_GET_LOG = 0x3b + IP_FW_RESETLOG = 0x37 + IP_FW_TABLE_ADD = 0x28 + IP_FW_TABLE_DEL = 0x29 + IP_FW_TABLE_FLUSH = 0x2a + IP_FW_TABLE_GETSIZE = 0x2b + IP_FW_TABLE_LIST = 0x2c + IP_FW_ZERO = 0x35 + IP_HDRINCL = 0x2 + IP_IPSEC_POLICY = 0x15 + IP_MAXPACKET = 0xffff + IP_MAX_GROUP_SRC_FILTER = 0x200 + IP_MAX_MEMBERSHIPS = 0xfff + IP_MAX_SOCK_MUTE_FILTER = 0x80 + IP_MAX_SOCK_SRC_FILTER = 0x80 + IP_MAX_SOURCE_FILTER = 0x400 + IP_MF = 0x2000 + IP_MINTTL = 0x42 + IP_MIN_MEMBERSHIPS = 0x1f + IP_MSFILTER = 0x4a + IP_MSS = 0x240 + IP_MULTICAST_IF = 0x9 + IP_MULTICAST_LOOP = 0xb + IP_MULTICAST_TTL = 0xa + IP_MULTICAST_VIF = 0xe + IP_OFFMASK = 0x1fff + IP_ONESBCAST = 0x17 + IP_OPTIONS = 0x1 + IP_PORTRANGE = 0x13 + IP_PORTRANGE_DEFAULT = 0 + IP_PORTRANGE_HIGH = 0x1 + IP_PORTRANGE_LOW = 0x2 + IP_RECVDSTADDR = 0x7 + IP_RECVIF = 0x14 + IP_RECVOPTS = 0x5 + IP_RECVRETOPTS = 0x6 + IP_RECVTTL = 0x41 + IP_RETOPTS = 0x8 + IP_RF = 0x8000 + IP_RSVP_OFF = 0x10 + IP_RSVP_ON = 0xf + IP_RSVP_VIF_OFF = 0x12 + IP_RSVP_VIF_ON = 0x11 + IP_SENDSRCADDR = 0x7 + IP_TOS = 0x3 + IP_TTL = 0x4 + IP_UNBLOCK_SOURCE = 0x49 + MSG_COMPAT = 0x8000 + MSG_CTRUNC = 0x20 + MSG_DONTROUTE = 0x4 + MSG_DONTWAIT = 0x80 + MSG_EOF = 0x100 + MSG_EOR = 0x8 + MSG_NBIO = 0x4000 + MSG_NOSIGNAL = 0x20000 + MSG_NOTIFICATION = 0x2000 + MSG_OOB = 0x1 + MSG_PEEK = 0x2 + MSG_TRUNC = 0x10 + MSG_WAITALL = 0x40 + NET_RT_DUMP = 0x1 + NET_RT_FLAGS = 0x2 + NET_RT_IFLIST = 0x3 + NET_RT_IFMALIST = 0x4 + NET_RT_MAXID = 0x5 + O_ACCMODE = 0x3 + O_APPEND = 0x8 + O_ASYNC = 0x40 + O_CREAT = 0x200 + O_DIRECT = 0x10000 + O_DIRECTORY = 0x20000 + O_EXCL = 0x800 + O_EXEC = 0x40000 + O_EXLOCK = 0x20 + O_FSYNC = 0x80 + O_NDELAY = 0x4 + O_NOCTTY = 0x8000 + O_NOFOLLOW = 0x100 + O_NONBLOCK = 0x4 + O_RDONLY = 0 + O_RDWR = 0x2 + O_SHLOCK = 0x10 + O_SYNC = 0x80 + O_TRUNC = 0x400 + O_TTY_INIT = 0x80000 + O_WRONLY = 0x1 + RTAX_AUTHOR = 0x6 + RTAX_BRD = 0x7 + RTAX_DST = 0 + RTAX_GATEWAY = 0x1 + RTAX_GENMASK = 0x3 + RTAX_IFA = 0x5 + RTAX_IFP = 0x4 + RTAX_MAX = 0x8 + RTAX_NETMASK = 0x2 + RTA_AUTHOR = 0x40 + RTA_BRD = 0x80 + RTA_DST = 0x1 + RTA_GATEWAY = 0x2 + RTA_GENMASK = 0x8 + RTA_IFA = 0x20 + RTA_IFP = 0x10 + RTA_NETMASK = 0x4 + RTF_BLACKHOLE = 0x1000 + RTF_BROADCAST = 0x400000 + RTF_DONE = 0x40 + RTF_DYNAMIC = 0x10 + RTF_FMASK = 0x1004d808 + RTF_GATEWAY = 0x2 + RTF_HOST = 0x4 + RTF_LLDATA = 0x400 + RTF_LLINFO = 0x400 + RTF_LOCAL = 0x200000 + RTF_MODIFIED = 0x20 + RTF_MULTICAST = 0x800000 + RTF_PINNED = 0x100000 + RTF_PRCLONING = 0x10000 + RTF_PROTO1 = 0x8000 + RTF_PROTO2 = 0x4000 + RTF_PROTO3 = 0x40000 + RTF_REJECT = 0x8 + RTF_RNH_LOCKED = 0x40000000 + RTF_STATIC = 0x800 + RTF_STICKY = 0x10000000 + RTF_UP = 0x1 + RTF_XRESOLVE = 0x200 + RTM_ADD = 0x1 + RTM_CHANGE = 0x3 + RTM_DELADDR = 0xd + RTM_DELETE = 0x2 + RTM_DELMADDR = 0x10 + RTM_GET = 0x4 + RTM_IEEE80211 = 0x12 + RTM_IFANNOUNCE = 0x11 + RTM_IFINFO = 0xe + RTM_LOCK = 0x8 + RTM_LOSING = 0x5 + RTM_MISS = 0x7 + RTM_NEWADDR = 0xc + RTM_NEWMADDR = 0xf + RTM_OLDADD = 0x9 + RTM_OLDDEL = 0xa + RTM_REDIRECT = 0x6 + RTM_RESOLVE = 0xb + RTM_RTTUNIT = 0xf4240 + RTM_VERSION = 0x5 + RTV_EXPIRE = 0x4 + RTV_HOPCOUNT = 0x2 + RTV_MTU = 0x1 + RTV_RPIPE = 0x8 + RTV_RTT = 0x40 + RTV_RTTVAR = 0x80 + RTV_SPIPE = 0x10 + RTV_SSTHRESH = 0x20 + RTV_WEIGHT = 0x100 + SCM_BINTIME = 0x4 + SCM_CREDS = 0x3 + SCM_RIGHTS = 0x1 + SCM_TIMESTAMP = 0x2 + SHUT_RD = 0 + SHUT_RDWR = 0x2 + SHUT_WR = 0x1 + SIGABRT = 0x6 + SIGALRM = 0xe + SIGBUS = 0xa + SIGCHLD = 0x14 + SIGCONT = 0x13 + SIGEMT = 0x7 + SIGFPE = 0x8 + SIGHUP = 0x1 + SIGILL = 0x4 + SIGINFO = 0x1d + SIGINT = 0x2 + SIGIO = 0x17 + SIGIOT = 0x6 + SIGKILL = 0x9 + SIGLWP = 0x20 + SIGPIPE = 0xd + SIGPROF = 0x1b + SIGQUIT = 0x3 + SIGSEGV = 0xb + SIGSTOP = 0x11 + SIGSYS = 0xc + SIGTERM = 0xf + SIGTHR = 0x20 + SIGTRAP = 0x5 + SIGTSTP = 0x12 + SIGTTIN = 0x15 + SIGTTOU = 0x16 + SIGURG = 0x10 + SIGUSR1 = 0x1e + SIGUSR2 = 0x1f + SIGVTALRM = 0x1a + SIGWINCH = 0x1c + SIGXCPU = 0x18 + SIGXFSZ = 0x19 + SIOCADDMULTI = 0x80206931 + SIOCADDRT = 0x8030720a + SIOCAIFADDR = 0x8040691a + SIOCAIFGROUP = 0x80246987 + SIOCALIFADDR = 0x8118691b + SIOCATMARK = 0x40047307 + SIOCDELMULTI = 0x80206932 + SIOCDELRT = 0x8030720b + SIOCDIFADDR = 0x80206919 + SIOCDIFGROUP = 0x80246989 + SIOCDIFPHYADDR = 0x80206949 + SIOCDLIFADDR = 0x8118691d + SIOCGDRVSPEC = 0xc01c697b + SIOCGETSGCNT = 0xc0147210 + SIOCGETVIFCNT = 0xc014720f + SIOCGHIWAT = 0x40047301 + SIOCGIFADDR = 0xc0206921 + SIOCGIFBRDADDR = 0xc0206923 + SIOCGIFCAP = 0xc020691f + SIOCGIFCONF = 0xc0086924 + SIOCGIFDESCR = 0xc020692a + SIOCGIFDSTADDR = 0xc0206922 + SIOCGIFFLAGS = 0xc0206911 + SIOCGIFGENERIC = 0xc020693a + SIOCGIFGMEMB = 0xc024698a + SIOCGIFGROUP = 0xc0246988 + SIOCGIFINDEX = 0xc0206920 + SIOCGIFMAC = 0xc0206926 + SIOCGIFMEDIA = 0xc0286938 + SIOCGIFMETRIC = 0xc0206917 + SIOCGIFMTU = 0xc0206933 + SIOCGIFNETMASK = 0xc0206925 + SIOCGIFPDSTADDR = 0xc0206948 + SIOCGIFPHYS = 0xc0206935 + SIOCGIFPSRCADDR = 0xc0206947 + SIOCGIFSTATUS = 0xc331693b + SIOCGLIFADDR = 0xc118691c + SIOCGLIFPHYADDR = 0xc118694b + SIOCGLOWAT = 0x40047303 + SIOCGPGRP = 0x40047309 + SIOCGPRIVATE_0 = 0xc0206950 + SIOCGPRIVATE_1 = 0xc0206951 + SIOCIFCREATE = 0xc020697a + SIOCIFCREATE2 = 0xc020697c + SIOCIFDESTROY = 0x80206979 + SIOCIFGCLONERS = 0xc00c6978 + SIOCSDRVSPEC = 0x801c697b + SIOCSHIWAT = 0x80047300 + SIOCSIFADDR = 0x8020690c + SIOCSIFBRDADDR = 0x80206913 + SIOCSIFCAP = 0x8020691e + SIOCSIFDESCR = 0x80206929 + SIOCSIFDSTADDR = 0x8020690e + SIOCSIFFLAGS = 0x80206910 + SIOCSIFGENERIC = 0x80206939 + SIOCSIFLLADDR = 0x8020693c + SIOCSIFMAC = 0x80206927 + SIOCSIFMEDIA = 0xc0206937 + SIOCSIFMETRIC = 0x80206918 + SIOCSIFMTU = 0x80206934 + SIOCSIFNAME = 0x80206928 + SIOCSIFNETMASK = 0x80206916 + SIOCSIFPHYADDR = 0x80406946 + SIOCSIFPHYS = 0x80206936 + SIOCSIFRVNET = 0xc020695b + SIOCSIFVNET = 0xc020695a + SIOCSLIFPHYADDR = 0x8118694a + SIOCSLOWAT = 0x80047302 + SIOCSPGRP = 0x80047308 + SOCK_DGRAM = 0x2 + SOCK_MAXADDRLEN = 0xff + SOCK_RAW = 0x3 + SOCK_RDM = 0x4 + SOCK_SEQPACKET = 0x5 + SOCK_STREAM = 0x1 + SOL_SOCKET = 0xffff + SOMAXCONN = 0x80 + SO_ACCEPTCONN = 0x2 + SO_ACCEPTFILTER = 0x1000 + SO_BINTIME = 0x2000 + SO_BROADCAST = 0x20 + SO_DEBUG = 0x1 + SO_DONTROUTE = 0x10 + SO_ERROR = 0x1007 + SO_KEEPALIVE = 0x8 + SO_LABEL = 0x1009 + SO_LINGER = 0x80 + SO_LISTENINCQLEN = 0x1013 + SO_LISTENQLEN = 0x1012 + SO_LISTENQLIMIT = 0x1011 + SO_NOSIGPIPE = 0x800 + SO_NO_DDP = 0x8000 + SO_NO_OFFLOAD = 0x4000 + SO_OOBINLINE = 0x100 + SO_PEERLABEL = 0x1010 + SO_RCVBUF = 0x1002 + SO_RCVLOWAT = 0x1004 + SO_RCVTIMEO = 0x1006 + SO_REUSEADDR = 0x4 + SO_REUSEPORT = 0x200 + SO_SETFIB = 0x1014 + SO_SNDBUF = 0x1001 + SO_SNDLOWAT = 0x1003 + SO_SNDTIMEO = 0x1005 + SO_TIMESTAMP = 0x400 + SO_TYPE = 0x1008 + SO_USELOOPBACK = 0x40 + TCP_CA_NAME_MAX = 0x10 + TCP_CONGESTION = 0x40 + TCP_INFO = 0x20 + TCP_MAXBURST = 0x4 + TCP_MAXHLEN = 0x3c + TCP_MAXOLEN = 0x28 + TCP_MAXSEG = 0x2 + TCP_MAXWIN = 0xffff + TCP_MAX_SACK = 0x4 + TCP_MAX_WINSHIFT = 0xe + TCP_MD5SIG = 0x10 + TCP_MINMSS = 0xd8 + TCP_MSS = 0x200 + TCP_NODELAY = 0x1 + TCP_NOOPT = 0x8 + TCP_NOPUSH = 0x4 + WCONTINUED = 0x4 + WCOREFLAG = 0x80 + WLINUXCLONE = 0x80000000 + WNOHANG = 0x1 + WNOWAIT = 0x8 + WSTOPPED = 0x2 + WUNTRACED = 0x2 ) // Types diff --git a/src/pkg/syscall/zerrors_freebsd_amd64.go b/src/pkg/syscall/zerrors_freebsd_amd64.go index 33fd3de1f..9b632ba93 100644 --- a/src/pkg/syscall/zerrors_freebsd_amd64.go +++ b/src/pkg/syscall/zerrors_freebsd_amd64.go @@ -9,1013 +9,1228 @@ package syscall // Constants const ( - AF_APPLETALK = 0x10 - AF_ARP = 0x23 - AF_ATM = 0x1e - AF_BLUETOOTH = 0x24 - AF_CCITT = 0xa - AF_CHAOS = 0x5 - AF_CNT = 0x15 - AF_COIP = 0x14 - AF_DATAKIT = 0x9 - AF_DECnet = 0xc - AF_DLI = 0xd - AF_E164 = 0x1a - AF_ECMA = 0x8 - AF_HYLINK = 0xf - AF_IEEE80211 = 0x25 - AF_IMPLINK = 0x3 - AF_INET = 0x2 - AF_INET6 = 0x1c - AF_IPX = 0x17 - AF_ISDN = 0x1a - AF_ISO = 0x7 - AF_LAT = 0xe - AF_LINK = 0x12 - AF_LOCAL = 0x1 - AF_MAX = 0x26 - AF_NATM = 0x1d - AF_NETBIOS = 0x6 - AF_NETGRAPH = 0x20 - AF_OSI = 0x7 - AF_PUP = 0x4 - AF_ROUTE = 0x11 - AF_SCLUSTER = 0x22 - AF_SIP = 0x18 - AF_SLOW = 0x21 - AF_SNA = 0xb - AF_UNIX = 0x1 - AF_UNSPEC = 0 - AF_VENDOR00 = 0x27 - AF_VENDOR01 = 0x29 - AF_VENDOR02 = 0x2b - AF_VENDOR03 = 0x2d - AF_VENDOR04 = 0x2f - AF_VENDOR05 = 0x31 - AF_VENDOR06 = 0x33 - AF_VENDOR07 = 0x35 - AF_VENDOR08 = 0x37 - AF_VENDOR09 = 0x39 - AF_VENDOR10 = 0x3b - AF_VENDOR11 = 0x3d - AF_VENDOR12 = 0x3f - AF_VENDOR13 = 0x41 - AF_VENDOR14 = 0x43 - AF_VENDOR15 = 0x45 - AF_VENDOR16 = 0x47 - AF_VENDOR17 = 0x49 - AF_VENDOR18 = 0x4b - AF_VENDOR19 = 0x4d - AF_VENDOR20 = 0x4f - AF_VENDOR21 = 0x51 - AF_VENDOR22 = 0x53 - AF_VENDOR23 = 0x55 - AF_VENDOR24 = 0x57 - AF_VENDOR25 = 0x59 - AF_VENDOR26 = 0x5b - AF_VENDOR27 = 0x5d - AF_VENDOR28 = 0x5f - AF_VENDOR29 = 0x61 - AF_VENDOR30 = 0x63 - AF_VENDOR31 = 0x65 - AF_VENDOR32 = 0x67 - AF_VENDOR33 = 0x69 - AF_VENDOR34 = 0x6b - AF_VENDOR35 = 0x6d - AF_VENDOR36 = 0x6f - AF_VENDOR37 = 0x71 - AF_VENDOR38 = 0x73 - AF_VENDOR39 = 0x75 - AF_VENDOR40 = 0x77 - AF_VENDOR41 = 0x79 - AF_VENDOR42 = 0x7b - AF_VENDOR43 = 0x7d - AF_VENDOR44 = 0x7f - AF_VENDOR45 = 0x81 - AF_VENDOR46 = 0x83 - AF_VENDOR47 = 0x85 - BIOCFEEDBACK = 0x8004427c - BIOCFLUSH = 0x20004268 - BIOCGBLEN = 0x40044266 - BIOCGDIRECTION = 0x40044276 - BIOCGDLT = 0x4004426a - BIOCGDLTLIST = 0xc0104279 - BIOCGETBUFMODE = 0x4004427d - BIOCGETIF = 0x4020426b - BIOCGETZMAX = 0x4008427f - BIOCGHDRCMPLT = 0x40044274 - BIOCGRSIG = 0x40044272 - BIOCGRTIMEOUT = 0x4010426e - BIOCGSEESENT = 0x40044276 - BIOCGSTATS = 0x4008426f - BIOCIMMEDIATE = 0x80044270 - BIOCLOCK = 0x2000427a - BIOCPROMISC = 0x20004269 - BIOCROTZBUF = 0x40184280 - BIOCSBLEN = 0xc0044266 - BIOCSDIRECTION = 0x80044277 - BIOCSDLT = 0x80044278 - BIOCSETBUFMODE = 0x8004427e - BIOCSETF = 0x80104267 - BIOCSETFNR = 0x80104282 - BIOCSETIF = 0x8020426c - BIOCSETWF = 0x8010427b - BIOCSETZBUF = 0x80184281 - BIOCSHDRCMPLT = 0x80044275 - BIOCSRSIG = 0x80044273 - BIOCSRTIMEOUT = 0x8010426d - BIOCSSEESENT = 0x80044277 - BIOCVERSION = 0x40044271 - BPF_A = 0x10 - BPF_ABS = 0x20 - BPF_ADD = 0 - BPF_ALIGNMENT = 0x8 - BPF_ALU = 0x4 - BPF_AND = 0x50 - BPF_B = 0x10 - BPF_BUFMODE_BUFFER = 0x1 - BPF_BUFMODE_ZBUF = 0x2 - BPF_DIV = 0x30 - BPF_H = 0x8 - BPF_IMM = 0 - BPF_IND = 0x40 - BPF_JA = 0 - BPF_JEQ = 0x10 - BPF_JGE = 0x30 - BPF_JGT = 0x20 - BPF_JMP = 0x5 - BPF_JSET = 0x40 - BPF_K = 0 - BPF_LD = 0 - BPF_LDX = 0x1 - BPF_LEN = 0x80 - BPF_LSH = 0x60 - BPF_MAJOR_VERSION = 0x1 - BPF_MAXBUFSIZE = 0x80000 - BPF_MAXINSNS = 0x200 - BPF_MEM = 0x60 - BPF_MEMWORDS = 0x10 - BPF_MINBUFSIZE = 0x20 - BPF_MINOR_VERSION = 0x1 - BPF_MISC = 0x7 - BPF_MSH = 0xa0 - BPF_MUL = 0x20 - BPF_NEG = 0x80 - BPF_OR = 0x40 - BPF_RELEASE = 0x30bb6 - BPF_RET = 0x6 - BPF_RSH = 0x70 - BPF_ST = 0x2 - BPF_STX = 0x3 - BPF_SUB = 0x10 - BPF_TAX = 0 - BPF_TXA = 0x80 - BPF_W = 0 - BPF_X = 0x8 - CTL_MAXNAME = 0x18 - CTL_NET = 0x4 - DLT_A429 = 0xb8 - DLT_A653_ICM = 0xb9 - DLT_AIRONET_HEADER = 0x78 - DLT_APPLE_IP_OVER_IEEE1394 = 0x8a - DLT_ARCNET = 0x7 - DLT_ARCNET_LINUX = 0x81 - DLT_ATM_CLIP = 0x13 - DLT_ATM_RFC1483 = 0xb - DLT_AURORA = 0x7e - DLT_AX25 = 0x3 - DLT_AX25_KISS = 0xca - DLT_BACNET_MS_TP = 0xa5 - DLT_BLUETOOTH_HCI_H4 = 0xbb - DLT_BLUETOOTH_HCI_H4_WITH_PHDR = 0xc9 - DLT_CAN20B = 0xbe - DLT_CHAOS = 0x5 - DLT_CHDLC = 0x68 - DLT_CISCO_IOS = 0x76 - DLT_C_HDLC = 0x68 - DLT_C_HDLC_WITH_DIR = 0xcd - DLT_DOCSIS = 0x8f - DLT_ECONET = 0x73 - DLT_EN10MB = 0x1 - DLT_EN3MB = 0x2 - DLT_ENC = 0x6d - DLT_ERF = 0xc5 - DLT_ERF_ETH = 0xaf - DLT_ERF_POS = 0xb0 - DLT_FDDI = 0xa - DLT_FLEXRAY = 0xd2 - DLT_FRELAY = 0x6b - DLT_FRELAY_WITH_DIR = 0xce - DLT_GCOM_SERIAL = 0xad - DLT_GCOM_T1E1 = 0xac - DLT_GPF_F = 0xab - DLT_GPF_T = 0xaa - DLT_GPRS_LLC = 0xa9 - DLT_HHDLC = 0x79 - DLT_IBM_SN = 0x92 - DLT_IBM_SP = 0x91 - DLT_IEEE802 = 0x6 - DLT_IEEE802_11 = 0x69 - DLT_IEEE802_11_RADIO = 0x7f - DLT_IEEE802_11_RADIO_AVS = 0xa3 - DLT_IEEE802_15_4 = 0xc3 - DLT_IEEE802_15_4_LINUX = 0xbf - DLT_IEEE802_15_4_NONASK_PHY = 0xd7 - DLT_IEEE802_16_MAC_CPS = 0xbc - DLT_IEEE802_16_MAC_CPS_RADIO = 0xc1 - DLT_IPFILTER = 0x74 - DLT_IPMB = 0xc7 - DLT_IPMB_LINUX = 0xd1 - DLT_IP_OVER_FC = 0x7a - DLT_JUNIPER_ATM1 = 0x89 - DLT_JUNIPER_ATM2 = 0x87 - DLT_JUNIPER_CHDLC = 0xb5 - DLT_JUNIPER_ES = 0x84 - DLT_JUNIPER_ETHER = 0xb2 - DLT_JUNIPER_FRELAY = 0xb4 - DLT_JUNIPER_GGSN = 0x85 - DLT_JUNIPER_ISM = 0xc2 - DLT_JUNIPER_MFR = 0x86 - DLT_JUNIPER_MLFR = 0x83 - DLT_JUNIPER_MLPPP = 0x82 - DLT_JUNIPER_MONITOR = 0xa4 - DLT_JUNIPER_PIC_PEER = 0xae - DLT_JUNIPER_PPP = 0xb3 - DLT_JUNIPER_PPPOE = 0xa7 - DLT_JUNIPER_PPPOE_ATM = 0xa8 - DLT_JUNIPER_SERVICES = 0x88 - DLT_JUNIPER_ST = 0xc8 - DLT_JUNIPER_VP = 0xb7 - DLT_LAPB_WITH_DIR = 0xcf - DLT_LAPD = 0xcb - DLT_LIN = 0xd4 - DLT_LINUX_IRDA = 0x90 - DLT_LINUX_LAPD = 0xb1 - DLT_LINUX_PPP_WITHDIRECTION = 0xa6 - DLT_LINUX_SLL = 0x71 - DLT_LOOP = 0x6c - DLT_LTALK = 0x72 - DLT_MFR = 0xb6 - DLT_MOST = 0xd3 - DLT_MTP2 = 0x8c - DLT_MTP2_WITH_PHDR = 0x8b - DLT_MTP3 = 0x8d - DLT_NULL = 0 - DLT_PCI_EXP = 0x7d - DLT_PFLOG = 0x75 - DLT_PFSYNC = 0x79 - DLT_PPI = 0xc0 - DLT_PPP = 0x9 - DLT_PPP_BSDOS = 0x10 - DLT_PPP_ETHER = 0x33 - DLT_PPP_PPPD = 0xa6 - DLT_PPP_SERIAL = 0x32 - DLT_PPP_WITH_DIR = 0xcc - DLT_PPP_WITH_DIRECTION = 0xa6 - DLT_PRISM_HEADER = 0x77 - DLT_PRONET = 0x4 - DLT_RAIF1 = 0xc6 - DLT_RAW = 0xc - DLT_RIO = 0x7c - DLT_SCCP = 0x8e - DLT_SITA = 0xc4 - DLT_SLIP = 0x8 - DLT_SLIP_BSDOS = 0xf - DLT_SUNATM = 0x7b - DLT_SYMANTEC_FIREWALL = 0x63 - DLT_TZSP = 0x80 - DLT_USB = 0xba - DLT_USB_LINUX = 0xbd - DLT_USER0 = 0x93 - DLT_USER1 = 0x94 - DLT_USER10 = 0x9d - DLT_USER11 = 0x9e - DLT_USER12 = 0x9f - DLT_USER13 = 0xa0 - DLT_USER14 = 0xa1 - DLT_USER15 = 0xa2 - DLT_USER2 = 0x95 - DLT_USER3 = 0x96 - DLT_USER4 = 0x97 - DLT_USER5 = 0x98 - DLT_USER6 = 0x99 - DLT_USER7 = 0x9a - DLT_USER8 = 0x9b - DLT_USER9 = 0x9c - DLT_X2E_SERIAL = 0xd5 - DLT_X2E_XORAYA = 0xd6 - E2BIG = 0x7 - EACCES = 0xd - EADDRINUSE = 0x30 - EADDRNOTAVAIL = 0x31 - EAFNOSUPPORT = 0x2f - EAGAIN = 0x23 - EALREADY = 0x25 - EAUTH = 0x50 - EBADF = 0x9 - EBADMSG = 0x59 - EBADRPC = 0x48 - EBUSY = 0x10 - ECANCELED = 0x55 - ECHILD = 0xa - ECONNABORTED = 0x35 - ECONNREFUSED = 0x3d - ECONNRESET = 0x36 - EDEADLK = 0xb - EDESTADDRREQ = 0x27 - EDOM = 0x21 - EDOOFUS = 0x58 - EDQUOT = 0x45 - EEXIST = 0x11 - EFAULT = 0xe - EFBIG = 0x1b - EFTYPE = 0x4f - EHOSTDOWN = 0x40 - EHOSTUNREACH = 0x41 - EIDRM = 0x52 - EILSEQ = 0x56 - EINPROGRESS = 0x24 - EINTR = 0x4 - EINVAL = 0x16 - EIO = 0x5 - EISCONN = 0x38 - EISDIR = 0x15 - ELAST = 0x5d - ELOOP = 0x3e - EMFILE = 0x18 - EMLINK = 0x1f - EMSGSIZE = 0x28 - EMULTIHOP = 0x5a - ENAMETOOLONG = 0x3f - ENEEDAUTH = 0x51 - ENETDOWN = 0x32 - ENETRESET = 0x34 - ENETUNREACH = 0x33 - ENFILE = 0x17 - ENOATTR = 0x57 - ENOBUFS = 0x37 - ENODEV = 0x13 - ENOENT = 0x2 - ENOEXEC = 0x8 - ENOLCK = 0x4d - ENOLINK = 0x5b - ENOMEM = 0xc - ENOMSG = 0x53 - ENOPROTOOPT = 0x2a - ENOSPC = 0x1c - ENOSYS = 0x4e - ENOTBLK = 0xf - ENOTCAPABLE = 0x5d - ENOTCONN = 0x39 - ENOTDIR = 0x14 - ENOTEMPTY = 0x42 - ENOTSOCK = 0x26 - ENOTSUP = 0x2d - ENOTTY = 0x19 - ENXIO = 0x6 - EOPNOTSUPP = 0x2d - EOVERFLOW = 0x54 - EPERM = 0x1 - EPFNOSUPPORT = 0x2e - EPIPE = 0x20 - EPROCLIM = 0x43 - EPROCUNAVAIL = 0x4c - EPROGMISMATCH = 0x4b - EPROGUNAVAIL = 0x4a - EPROTO = 0x5c - EPROTONOSUPPORT = 0x2b - EPROTOTYPE = 0x29 - ERANGE = 0x22 - EREMOTE = 0x47 - EROFS = 0x1e - ERPCMISMATCH = 0x49 - ESHUTDOWN = 0x3a - ESOCKTNOSUPPORT = 0x2c - ESPIPE = 0x1d - ESRCH = 0x3 - ESTALE = 0x46 - ETIMEDOUT = 0x3c - ETOOMANYREFS = 0x3b - ETXTBSY = 0x1a - EUSERS = 0x44 - EVFILT_AIO = -0x3 - EVFILT_FS = -0x9 - EVFILT_LIO = -0xa - EVFILT_PROC = -0x5 - EVFILT_READ = -0x1 - EVFILT_SIGNAL = -0x6 - EVFILT_SYSCOUNT = 0xb - EVFILT_TIMER = -0x7 - EVFILT_USER = -0xb - EVFILT_VNODE = -0x4 - EVFILT_WRITE = -0x2 - EV_ADD = 0x1 - EV_CLEAR = 0x20 - EV_DELETE = 0x2 - EV_DISABLE = 0x8 - EV_DISPATCH = 0x80 - EV_ENABLE = 0x4 - EV_EOF = 0x8000 - EV_ERROR = 0x4000 - EV_FLAG1 = 0x2000 - EV_ONESHOT = 0x10 - EV_RECEIPT = 0x40 - EV_SYSFLAGS = 0xf000 - EWOULDBLOCK = 0x23 - EXDEV = 0x12 - FD_CLOEXEC = 0x1 - FD_SETSIZE = 0x400 - F_CANCEL = 0x5 - F_DUP2FD = 0xa - F_DUPFD = 0 - F_GETFD = 0x1 - F_GETFL = 0x3 - F_GETLK = 0xb - F_GETOWN = 0x5 - F_OGETLK = 0x7 - F_OSETLK = 0x8 - F_OSETLKW = 0x9 - F_RDAHEAD = 0x10 - F_RDLCK = 0x1 - F_READAHEAD = 0xf - F_SETFD = 0x2 - F_SETFL = 0x4 - F_SETLK = 0xc - F_SETLKW = 0xd - F_SETLK_REMOTE = 0xe - F_SETOWN = 0x6 - F_UNLCK = 0x2 - F_UNLCKSYS = 0x4 - F_WRLCK = 0x3 - IFF_ALLMULTI = 0x200 - IFF_ALTPHYS = 0x4000 - IFF_BROADCAST = 0x2 - IFF_CANTCHANGE = 0x208f72 - IFF_DEBUG = 0x4 - IFF_DRV_OACTIVE = 0x400 - IFF_DRV_RUNNING = 0x40 - IFF_DYING = 0x200000 - IFF_LINK0 = 0x1000 - IFF_LINK1 = 0x2000 - IFF_LINK2 = 0x4000 - IFF_LOOPBACK = 0x8 - IFF_MONITOR = 0x40000 - IFF_MULTICAST = 0x8000 - IFF_NOARP = 0x80 - IFF_OACTIVE = 0x400 - IFF_POINTOPOINT = 0x10 - IFF_PPROMISC = 0x20000 - IFF_PROMISC = 0x100 - IFF_RENAMING = 0x400000 - IFF_RUNNING = 0x40 - IFF_SIMPLEX = 0x800 - IFF_SMART = 0x20 - IFF_STATICARP = 0x80000 - IFF_UP = 0x1 - IFNAMSIZ = 0x10 - IN_CLASSA_HOST = 0xffffff - IN_CLASSA_MAX = 0x80 - IN_CLASSA_NET = 0xff000000 - IN_CLASSA_NSHIFT = 0x18 - IN_CLASSB_HOST = 0xffff - IN_CLASSB_MAX = 0x10000 - IN_CLASSB_NET = 0xffff0000 - IN_CLASSB_NSHIFT = 0x10 - IN_CLASSC_HOST = 0xff - IN_CLASSC_NET = 0xffffff00 - IN_CLASSC_NSHIFT = 0x8 - IN_CLASSD_HOST = 0xfffffff - IN_CLASSD_NET = 0xf0000000 - IN_CLASSD_NSHIFT = 0x1c - IN_LOOPBACKNET = 0x7f - IPPROTO_3PC = 0x22 - IPPROTO_ADFS = 0x44 - IPPROTO_AH = 0x33 - IPPROTO_AHIP = 0x3d - IPPROTO_APES = 0x63 - IPPROTO_ARGUS = 0xd - IPPROTO_AX25 = 0x5d - IPPROTO_BHA = 0x31 - IPPROTO_BLT = 0x1e - IPPROTO_BRSATMON = 0x4c - IPPROTO_CARP = 0x70 - IPPROTO_CFTP = 0x3e - IPPROTO_CHAOS = 0x10 - IPPROTO_CMTP = 0x26 - IPPROTO_CPHB = 0x49 - IPPROTO_CPNX = 0x48 - IPPROTO_DDP = 0x25 - IPPROTO_DGP = 0x56 - IPPROTO_DIVERT = 0x102 - IPPROTO_DONE = 0x101 - IPPROTO_DSTOPTS = 0x3c - IPPROTO_EGP = 0x8 - IPPROTO_EMCON = 0xe - IPPROTO_ENCAP = 0x62 - IPPROTO_EON = 0x50 - IPPROTO_ESP = 0x32 - IPPROTO_ETHERIP = 0x61 - IPPROTO_FRAGMENT = 0x2c - IPPROTO_GGP = 0x3 - IPPROTO_GMTP = 0x64 - IPPROTO_GRE = 0x2f - IPPROTO_HELLO = 0x3f - IPPROTO_HMP = 0x14 - IPPROTO_HOPOPTS = 0 - IPPROTO_ICMP = 0x1 - IPPROTO_ICMPV6 = 0x3a - IPPROTO_IDP = 0x16 - IPPROTO_IDPR = 0x23 - IPPROTO_IDRP = 0x2d - IPPROTO_IGMP = 0x2 - IPPROTO_IGP = 0x55 - IPPROTO_IGRP = 0x58 - IPPROTO_IL = 0x28 - IPPROTO_INLSP = 0x34 - IPPROTO_INP = 0x20 - IPPROTO_IP = 0 - IPPROTO_IPCOMP = 0x6c - IPPROTO_IPCV = 0x47 - IPPROTO_IPEIP = 0x5e - IPPROTO_IPIP = 0x4 - IPPROTO_IPPC = 0x43 - IPPROTO_IPV4 = 0x4 - IPPROTO_IPV6 = 0x29 - IPPROTO_IRTP = 0x1c - IPPROTO_KRYPTOLAN = 0x41 - IPPROTO_LARP = 0x5b - IPPROTO_LEAF1 = 0x19 - IPPROTO_LEAF2 = 0x1a - IPPROTO_MAX = 0x100 - IPPROTO_MAXID = 0x34 - IPPROTO_MEAS = 0x13 - IPPROTO_MHRP = 0x30 - IPPROTO_MICP = 0x5f - IPPROTO_MOBILE = 0x37 - IPPROTO_MTP = 0x5c - IPPROTO_MUX = 0x12 - IPPROTO_ND = 0x4d - IPPROTO_NHRP = 0x36 - IPPROTO_NONE = 0x3b - IPPROTO_NSP = 0x1f - IPPROTO_NVPII = 0xb - IPPROTO_OLD_DIVERT = 0xfe - IPPROTO_OSPFIGP = 0x59 - IPPROTO_PFSYNC = 0xf0 - IPPROTO_PGM = 0x71 - IPPROTO_PIGP = 0x9 - IPPROTO_PIM = 0x67 - IPPROTO_PRM = 0x15 - IPPROTO_PUP = 0xc - IPPROTO_PVP = 0x4b - IPPROTO_RAW = 0xff - IPPROTO_RCCMON = 0xa - IPPROTO_RDP = 0x1b - IPPROTO_ROUTING = 0x2b - IPPROTO_RSVP = 0x2e - IPPROTO_RVD = 0x42 - IPPROTO_SATEXPAK = 0x40 - IPPROTO_SATMON = 0x45 - IPPROTO_SCCSP = 0x60 - IPPROTO_SCTP = 0x84 - IPPROTO_SDRP = 0x2a - IPPROTO_SEP = 0x21 - IPPROTO_SKIP = 0x39 - IPPROTO_SPACER = 0x7fff - IPPROTO_SRPC = 0x5a - IPPROTO_ST = 0x7 - IPPROTO_SVMTP = 0x52 - IPPROTO_SWIPE = 0x35 - IPPROTO_TCF = 0x57 - IPPROTO_TCP = 0x6 - IPPROTO_TLSP = 0x38 - IPPROTO_TP = 0x1d - IPPROTO_TPXX = 0x27 - IPPROTO_TRUNK1 = 0x17 - IPPROTO_TRUNK2 = 0x18 - IPPROTO_TTP = 0x54 - IPPROTO_UDP = 0x11 - IPPROTO_VINES = 0x53 - IPPROTO_VISA = 0x46 - IPPROTO_VMTP = 0x51 - IPPROTO_WBEXPAK = 0x4f - IPPROTO_WBMON = 0x4e - IPPROTO_WSN = 0x4a - IPPROTO_XNET = 0xf - IPPROTO_XTP = 0x24 - IPV6_AUTOFLOWLABEL = 0x3b - IPV6_BINDANY = 0x40 - IPV6_BINDV6ONLY = 0x1b - IPV6_CHECKSUM = 0x1a - IPV6_DEFAULT_MULTICAST_HOPS = 0x1 - IPV6_DEFAULT_MULTICAST_LOOP = 0x1 - IPV6_DEFHLIM = 0x40 - IPV6_DONTFRAG = 0x3e - IPV6_DSTOPTS = 0x32 - IPV6_FAITH = 0x1d - IPV6_FLOWINFO_MASK = 0xffffff0f - IPV6_FLOWLABEL_MASK = 0xffff0f00 - IPV6_FRAGTTL = 0x78 - IPV6_FW_ADD = 0x1e - IPV6_FW_DEL = 0x1f - IPV6_FW_FLUSH = 0x20 - IPV6_FW_GET = 0x22 - IPV6_FW_ZERO = 0x21 - IPV6_HLIMDEC = 0x1 - IPV6_HOPLIMIT = 0x2f - IPV6_HOPOPTS = 0x31 - IPV6_IPSEC_POLICY = 0x1c - IPV6_JOIN_GROUP = 0xc - IPV6_LEAVE_GROUP = 0xd - IPV6_MAXHLIM = 0xff - IPV6_MAXOPTHDR = 0x800 - IPV6_MAXPACKET = 0xffff - IPV6_MAX_GROUP_SRC_FILTER = 0x200 - IPV6_MAX_MEMBERSHIPS = 0xfff - IPV6_MAX_SOCK_SRC_FILTER = 0x80 - IPV6_MIN_MEMBERSHIPS = 0x1f - IPV6_MMTU = 0x500 - IPV6_MSFILTER = 0x4a - IPV6_MULTICAST_HOPS = 0xa - IPV6_MULTICAST_IF = 0x9 - IPV6_MULTICAST_LOOP = 0xb - IPV6_NEXTHOP = 0x30 - IPV6_PATHMTU = 0x2c - IPV6_PKTINFO = 0x2e - IPV6_PORTRANGE = 0xe - IPV6_PORTRANGE_DEFAULT = 0 - IPV6_PORTRANGE_HIGH = 0x1 - IPV6_PORTRANGE_LOW = 0x2 - IPV6_PREFER_TEMPADDR = 0x3f - IPV6_RECVDSTOPTS = 0x28 - IPV6_RECVHOPLIMIT = 0x25 - IPV6_RECVHOPOPTS = 0x27 - IPV6_RECVPATHMTU = 0x2b - IPV6_RECVPKTINFO = 0x24 - IPV6_RECVRTHDR = 0x26 - IPV6_RECVTCLASS = 0x39 - IPV6_RTHDR = 0x33 - IPV6_RTHDRDSTOPTS = 0x23 - IPV6_RTHDR_LOOSE = 0 - IPV6_RTHDR_STRICT = 0x1 - IPV6_RTHDR_TYPE_0 = 0 - IPV6_SOCKOPT_RESERVED1 = 0x3 - IPV6_TCLASS = 0x3d - IPV6_UNICAST_HOPS = 0x4 - IPV6_USE_MIN_MTU = 0x2a - IPV6_V6ONLY = 0x1b - IPV6_VERSION = 0x60 - IPV6_VERSION_MASK = 0xf0 - IP_ADD_MEMBERSHIP = 0xc - IP_ADD_SOURCE_MEMBERSHIP = 0x46 - IP_BINDANY = 0x18 - IP_BLOCK_SOURCE = 0x48 - IP_DEFAULT_MULTICAST_LOOP = 0x1 - IP_DEFAULT_MULTICAST_TTL = 0x1 - IP_DF = 0x4000 - IP_DONTFRAG = 0x43 - IP_DROP_MEMBERSHIP = 0xd - IP_DROP_SOURCE_MEMBERSHIP = 0x47 - IP_DUMMYNET3 = 0x31 - IP_DUMMYNET_CONFIGURE = 0x3c - IP_DUMMYNET_DEL = 0x3d - IP_DUMMYNET_FLUSH = 0x3e - IP_DUMMYNET_GET = 0x40 - IP_FAITH = 0x16 - IP_FW3 = 0x30 - IP_FW_ADD = 0x32 - IP_FW_DEL = 0x33 - IP_FW_FLUSH = 0x34 - IP_FW_GET = 0x36 - IP_FW_NAT_CFG = 0x38 - IP_FW_NAT_DEL = 0x39 - IP_FW_NAT_GET_CONFIG = 0x3a - IP_FW_NAT_GET_LOG = 0x3b - IP_FW_RESETLOG = 0x37 - IP_FW_TABLE_ADD = 0x28 - IP_FW_TABLE_DEL = 0x29 - IP_FW_TABLE_FLUSH = 0x2a - IP_FW_TABLE_GETSIZE = 0x2b - IP_FW_TABLE_LIST = 0x2c - IP_FW_ZERO = 0x35 - IP_HDRINCL = 0x2 - IP_IPSEC_POLICY = 0x15 - IP_MAXPACKET = 0xffff - IP_MAX_GROUP_SRC_FILTER = 0x200 - IP_MAX_MEMBERSHIPS = 0xfff - IP_MAX_SOCK_MUTE_FILTER = 0x80 - IP_MAX_SOCK_SRC_FILTER = 0x80 - IP_MAX_SOURCE_FILTER = 0x400 - IP_MF = 0x2000 - IP_MINTTL = 0x42 - IP_MIN_MEMBERSHIPS = 0x1f - IP_MSFILTER = 0x4a - IP_MSS = 0x240 - IP_MULTICAST_IF = 0x9 - IP_MULTICAST_LOOP = 0xb - IP_MULTICAST_TTL = 0xa - IP_MULTICAST_VIF = 0xe - IP_OFFMASK = 0x1fff - IP_ONESBCAST = 0x17 - IP_OPTIONS = 0x1 - IP_PORTRANGE = 0x13 - IP_PORTRANGE_DEFAULT = 0 - IP_PORTRANGE_HIGH = 0x1 - IP_PORTRANGE_LOW = 0x2 - IP_RECVDSTADDR = 0x7 - IP_RECVIF = 0x14 - IP_RECVOPTS = 0x5 - IP_RECVRETOPTS = 0x6 - IP_RECVTTL = 0x41 - IP_RETOPTS = 0x8 - IP_RF = 0x8000 - IP_RSVP_OFF = 0x10 - IP_RSVP_ON = 0xf - IP_RSVP_VIF_OFF = 0x12 - IP_RSVP_VIF_ON = 0x11 - IP_SENDSRCADDR = 0x7 - IP_TOS = 0x3 - IP_TTL = 0x4 - IP_UNBLOCK_SOURCE = 0x49 - MSG_COMPAT = 0x8000 - MSG_CTRUNC = 0x20 - MSG_DONTROUTE = 0x4 - MSG_DONTWAIT = 0x80 - MSG_EOF = 0x100 - MSG_EOR = 0x8 - MSG_NBIO = 0x4000 - MSG_NOSIGNAL = 0x20000 - MSG_NOTIFICATION = 0x2000 - MSG_OOB = 0x1 - MSG_PEEK = 0x2 - MSG_TRUNC = 0x10 - MSG_WAITALL = 0x40 - NET_RT_DUMP = 0x1 - NET_RT_FLAGS = 0x2 - NET_RT_IFLIST = 0x3 - NET_RT_IFMALIST = 0x4 - NET_RT_MAXID = 0x5 - O_ACCMODE = 0x3 - O_APPEND = 0x8 - O_ASYNC = 0x40 - O_CREAT = 0x200 - O_DIRECT = 0x10000 - O_DIRECTORY = 0x20000 - O_EXCL = 0x800 - O_EXEC = 0x40000 - O_EXLOCK = 0x20 - O_FSYNC = 0x80 - O_NDELAY = 0x4 - O_NOCTTY = 0x8000 - O_NOFOLLOW = 0x100 - O_NONBLOCK = 0x4 - O_RDONLY = 0 - O_RDWR = 0x2 - O_SHLOCK = 0x10 - O_SYNC = 0x80 - O_TRUNC = 0x400 - O_TTY_INIT = 0x80000 - O_WRONLY = 0x1 - RTAX_AUTHOR = 0x6 - RTAX_BRD = 0x7 - RTAX_DST = 0 - RTAX_GATEWAY = 0x1 - RTAX_GENMASK = 0x3 - RTAX_IFA = 0x5 - RTAX_IFP = 0x4 - RTAX_MAX = 0x8 - RTAX_NETMASK = 0x2 - RTA_AUTHOR = 0x40 - RTA_BRD = 0x80 - RTA_DST = 0x1 - RTA_GATEWAY = 0x2 - RTA_GENMASK = 0x8 - RTA_IFA = 0x20 - RTA_IFP = 0x10 - RTA_NETMASK = 0x4 - RTF_BLACKHOLE = 0x1000 - RTF_BROADCAST = 0x400000 - RTF_DONE = 0x40 - RTF_DYNAMIC = 0x10 - RTF_FMASK = 0x1004d808 - RTF_GATEWAY = 0x2 - RTF_HOST = 0x4 - RTF_LLDATA = 0x400 - RTF_LLINFO = 0x400 - RTF_LOCAL = 0x200000 - RTF_MODIFIED = 0x20 - RTF_MULTICAST = 0x800000 - RTF_PINNED = 0x100000 - RTF_PRCLONING = 0x10000 - RTF_PROTO1 = 0x8000 - RTF_PROTO2 = 0x4000 - RTF_PROTO3 = 0x40000 - RTF_REJECT = 0x8 - RTF_RNH_LOCKED = 0x40000000 - RTF_STATIC = 0x800 - RTF_STICKY = 0x10000000 - RTF_UP = 0x1 - RTF_XRESOLVE = 0x200 - RTM_ADD = 0x1 - RTM_CHANGE = 0x3 - RTM_DELADDR = 0xd - RTM_DELETE = 0x2 - RTM_DELMADDR = 0x10 - RTM_GET = 0x4 - RTM_IEEE80211 = 0x12 - RTM_IFANNOUNCE = 0x11 - RTM_IFINFO = 0xe - RTM_LOCK = 0x8 - RTM_LOSING = 0x5 - RTM_MISS = 0x7 - RTM_NEWADDR = 0xc - RTM_NEWMADDR = 0xf - RTM_OLDADD = 0x9 - RTM_OLDDEL = 0xa - RTM_REDIRECT = 0x6 - RTM_RESOLVE = 0xb - RTM_RTTUNIT = 0xf4240 - RTM_VERSION = 0x5 - RTV_EXPIRE = 0x4 - RTV_HOPCOUNT = 0x2 - RTV_MTU = 0x1 - RTV_RPIPE = 0x8 - RTV_RTT = 0x40 - RTV_RTTVAR = 0x80 - RTV_SPIPE = 0x10 - RTV_SSTHRESH = 0x20 - RTV_WEIGHT = 0x100 - SCM_BINTIME = 0x4 - SCM_CREDS = 0x3 - SCM_RIGHTS = 0x1 - SCM_TIMESTAMP = 0x2 - SHUT_RD = 0 - SHUT_RDWR = 0x2 - SHUT_WR = 0x1 - SIGABRT = 0x6 - SIGALRM = 0xe - SIGBUS = 0xa - SIGCHLD = 0x14 - SIGCONT = 0x13 - SIGEMT = 0x7 - SIGFPE = 0x8 - SIGHUP = 0x1 - SIGILL = 0x4 - SIGINFO = 0x1d - SIGINT = 0x2 - SIGIO = 0x17 - SIGIOT = 0x6 - SIGKILL = 0x9 - SIGLWP = 0x20 - SIGPIPE = 0xd - SIGPROF = 0x1b - SIGQUIT = 0x3 - SIGSEGV = 0xb - SIGSTOP = 0x11 - SIGSYS = 0xc - SIGTERM = 0xf - SIGTHR = 0x20 - SIGTRAP = 0x5 - SIGTSTP = 0x12 - SIGTTIN = 0x15 - SIGTTOU = 0x16 - SIGURG = 0x10 - SIGUSR1 = 0x1e - SIGUSR2 = 0x1f - SIGVTALRM = 0x1a - SIGWINCH = 0x1c - SIGXCPU = 0x18 - SIGXFSZ = 0x19 - SIOCADDMULTI = 0x80206931 - SIOCADDRT = 0x8040720a - SIOCAIFADDR = 0x8040691a - SIOCAIFGROUP = 0x80286987 - SIOCALIFADDR = 0x8118691b - SIOCATMARK = 0x40047307 - SIOCDELMULTI = 0x80206932 - SIOCDELRT = 0x8040720b - SIOCDIFADDR = 0x80206919 - SIOCDIFGROUP = 0x80286989 - SIOCDIFPHYADDR = 0x80206949 - SIOCDLIFADDR = 0x8118691d - SIOCGDRVSPEC = 0xc028697b - SIOCGETSGCNT = 0xc0207210 - SIOCGETVIFCNT = 0xc028720f - SIOCGHIWAT = 0x40047301 - SIOCGIFADDR = 0xc0206921 - SIOCGIFBRDADDR = 0xc0206923 - SIOCGIFCAP = 0xc020691f - SIOCGIFCONF = 0xc0106924 - SIOCGIFDESCR = 0xc020692a - SIOCGIFDSTADDR = 0xc0206922 - SIOCGIFFLAGS = 0xc0206911 - SIOCGIFGENERIC = 0xc020693a - SIOCGIFGMEMB = 0xc028698a - SIOCGIFGROUP = 0xc0286988 - SIOCGIFINDEX = 0xc0206920 - SIOCGIFMAC = 0xc0206926 - SIOCGIFMEDIA = 0xc0306938 - SIOCGIFMETRIC = 0xc0206917 - SIOCGIFMTU = 0xc0206933 - SIOCGIFNETMASK = 0xc0206925 - SIOCGIFPDSTADDR = 0xc0206948 - SIOCGIFPHYS = 0xc0206935 - SIOCGIFPSRCADDR = 0xc0206947 - SIOCGIFSTATUS = 0xc331693b - SIOCGLIFADDR = 0xc118691c - SIOCGLIFPHYADDR = 0xc118694b - SIOCGLOWAT = 0x40047303 - SIOCGPGRP = 0x40047309 - SIOCGPRIVATE_0 = 0xc0206950 - SIOCGPRIVATE_1 = 0xc0206951 - SIOCIFCREATE = 0xc020697a - SIOCIFCREATE2 = 0xc020697c - SIOCIFDESTROY = 0x80206979 - SIOCIFGCLONERS = 0xc0106978 - SIOCSDRVSPEC = 0x8028697b - SIOCSHIWAT = 0x80047300 - SIOCSIFADDR = 0x8020690c - SIOCSIFBRDADDR = 0x80206913 - SIOCSIFCAP = 0x8020691e - SIOCSIFDESCR = 0x80206929 - SIOCSIFDSTADDR = 0x8020690e - SIOCSIFFLAGS = 0x80206910 - SIOCSIFGENERIC = 0x80206939 - SIOCSIFLLADDR = 0x8020693c - SIOCSIFMAC = 0x80206927 - SIOCSIFMEDIA = 0xc0206937 - SIOCSIFMETRIC = 0x80206918 - SIOCSIFMTU = 0x80206934 - SIOCSIFNAME = 0x80206928 - SIOCSIFNETMASK = 0x80206916 - SIOCSIFPHYADDR = 0x80406946 - SIOCSIFPHYS = 0x80206936 - SIOCSIFRVNET = 0xc020695b - SIOCSIFVNET = 0xc020695a - SIOCSLIFPHYADDR = 0x8118694a - SIOCSLOWAT = 0x80047302 - SIOCSPGRP = 0x80047308 - SOCK_DGRAM = 0x2 - SOCK_MAXADDRLEN = 0xff - SOCK_RAW = 0x3 - SOCK_RDM = 0x4 - SOCK_SEQPACKET = 0x5 - SOCK_STREAM = 0x1 - SOL_SOCKET = 0xffff - SOMAXCONN = 0x80 - SO_ACCEPTCONN = 0x2 - SO_ACCEPTFILTER = 0x1000 - SO_BINTIME = 0x2000 - SO_BROADCAST = 0x20 - SO_DEBUG = 0x1 - SO_DONTROUTE = 0x10 - SO_ERROR = 0x1007 - SO_KEEPALIVE = 0x8 - SO_LABEL = 0x1009 - SO_LINGER = 0x80 - SO_LISTENINCQLEN = 0x1013 - SO_LISTENQLEN = 0x1012 - SO_LISTENQLIMIT = 0x1011 - SO_NOSIGPIPE = 0x800 - SO_NO_DDP = 0x8000 - SO_NO_OFFLOAD = 0x4000 - SO_OOBINLINE = 0x100 - SO_PEERLABEL = 0x1010 - SO_RCVBUF = 0x1002 - SO_RCVLOWAT = 0x1004 - SO_RCVTIMEO = 0x1006 - SO_REUSEADDR = 0x4 - SO_REUSEPORT = 0x200 - SO_SETFIB = 0x1014 - SO_SNDBUF = 0x1001 - SO_SNDLOWAT = 0x1003 - SO_SNDTIMEO = 0x1005 - SO_TIMESTAMP = 0x400 - SO_TYPE = 0x1008 - SO_USELOOPBACK = 0x40 - TCP_CA_NAME_MAX = 0x10 - TCP_CONGESTION = 0x40 - TCP_INFO = 0x20 - TCP_MAXBURST = 0x4 - TCP_MAXHLEN = 0x3c - TCP_MAXOLEN = 0x28 - TCP_MAXSEG = 0x2 - TCP_MAXWIN = 0xffff - TCP_MAX_SACK = 0x4 - TCP_MAX_WINSHIFT = 0xe - TCP_MD5SIG = 0x10 - TCP_MINMSS = 0xd8 - TCP_MSS = 0x200 - TCP_NODELAY = 0x1 - TCP_NOOPT = 0x8 - TCP_NOPUSH = 0x4 - WCONTINUED = 0x4 - WCOREFLAG = 0x80 - WLINUXCLONE = 0x80000000 - WNOHANG = 0x1 - WNOWAIT = 0x8 - WSTOPPED = 0x2 - WUNTRACED = 0x2 + AF_APPLETALK = 0x10 + AF_ARP = 0x23 + AF_ATM = 0x1e + AF_BLUETOOTH = 0x24 + AF_CCITT = 0xa + AF_CHAOS = 0x5 + AF_CNT = 0x15 + AF_COIP = 0x14 + AF_DATAKIT = 0x9 + AF_DECnet = 0xc + AF_DLI = 0xd + AF_E164 = 0x1a + AF_ECMA = 0x8 + AF_HYLINK = 0xf + AF_IEEE80211 = 0x25 + AF_IMPLINK = 0x3 + AF_INET = 0x2 + AF_INET6 = 0x1c + AF_IPX = 0x17 + AF_ISDN = 0x1a + AF_ISO = 0x7 + AF_LAT = 0xe + AF_LINK = 0x12 + AF_LOCAL = 0x1 + AF_MAX = 0x26 + AF_NATM = 0x1d + AF_NETBIOS = 0x6 + AF_NETGRAPH = 0x20 + AF_OSI = 0x7 + AF_PUP = 0x4 + AF_ROUTE = 0x11 + AF_SCLUSTER = 0x22 + AF_SIP = 0x18 + AF_SLOW = 0x21 + AF_SNA = 0xb + AF_UNIX = 0x1 + AF_UNSPEC = 0 + AF_VENDOR00 = 0x27 + AF_VENDOR01 = 0x29 + AF_VENDOR02 = 0x2b + AF_VENDOR03 = 0x2d + AF_VENDOR04 = 0x2f + AF_VENDOR05 = 0x31 + AF_VENDOR06 = 0x33 + AF_VENDOR07 = 0x35 + AF_VENDOR08 = 0x37 + AF_VENDOR09 = 0x39 + AF_VENDOR10 = 0x3b + AF_VENDOR11 = 0x3d + AF_VENDOR12 = 0x3f + AF_VENDOR13 = 0x41 + AF_VENDOR14 = 0x43 + AF_VENDOR15 = 0x45 + AF_VENDOR16 = 0x47 + AF_VENDOR17 = 0x49 + AF_VENDOR18 = 0x4b + AF_VENDOR19 = 0x4d + AF_VENDOR20 = 0x4f + AF_VENDOR21 = 0x51 + AF_VENDOR22 = 0x53 + AF_VENDOR23 = 0x55 + AF_VENDOR24 = 0x57 + AF_VENDOR25 = 0x59 + AF_VENDOR26 = 0x5b + AF_VENDOR27 = 0x5d + AF_VENDOR28 = 0x5f + AF_VENDOR29 = 0x61 + AF_VENDOR30 = 0x63 + AF_VENDOR31 = 0x65 + AF_VENDOR32 = 0x67 + AF_VENDOR33 = 0x69 + AF_VENDOR34 = 0x6b + AF_VENDOR35 = 0x6d + AF_VENDOR36 = 0x6f + AF_VENDOR37 = 0x71 + AF_VENDOR38 = 0x73 + AF_VENDOR39 = 0x75 + AF_VENDOR40 = 0x77 + AF_VENDOR41 = 0x79 + AF_VENDOR42 = 0x7b + AF_VENDOR43 = 0x7d + AF_VENDOR44 = 0x7f + AF_VENDOR45 = 0x81 + AF_VENDOR46 = 0x83 + AF_VENDOR47 = 0x85 + BIOCFEEDBACK = 0x8004427c + BIOCFLUSH = 0x20004268 + BIOCGBLEN = 0x40044266 + BIOCGDIRECTION = 0x40044276 + BIOCGDLT = 0x4004426a + BIOCGDLTLIST = 0xc0104279 + BIOCGETBUFMODE = 0x4004427d + BIOCGETIF = 0x4020426b + BIOCGETZMAX = 0x4008427f + BIOCGHDRCMPLT = 0x40044274 + BIOCGRSIG = 0x40044272 + BIOCGRTIMEOUT = 0x4010426e + BIOCGSEESENT = 0x40044276 + BIOCGSTATS = 0x4008426f + BIOCIMMEDIATE = 0x80044270 + BIOCLOCK = 0x2000427a + BIOCPROMISC = 0x20004269 + BIOCROTZBUF = 0x40184280 + BIOCSBLEN = 0xc0044266 + BIOCSDIRECTION = 0x80044277 + BIOCSDLT = 0x80044278 + BIOCSETBUFMODE = 0x8004427e + BIOCSETF = 0x80104267 + BIOCSETFNR = 0x80104282 + BIOCSETIF = 0x8020426c + BIOCSETWF = 0x8010427b + BIOCSETZBUF = 0x80184281 + BIOCSHDRCMPLT = 0x80044275 + BIOCSRSIG = 0x80044273 + BIOCSRTIMEOUT = 0x8010426d + BIOCSSEESENT = 0x80044277 + BIOCVERSION = 0x40044271 + BPF_A = 0x10 + BPF_ABS = 0x20 + BPF_ADD = 0 + BPF_ALIGNMENT = 0x8 + BPF_ALU = 0x4 + BPF_AND = 0x50 + BPF_B = 0x10 + BPF_BUFMODE_BUFFER = 0x1 + BPF_BUFMODE_ZBUF = 0x2 + BPF_DIV = 0x30 + BPF_H = 0x8 + BPF_IMM = 0 + BPF_IND = 0x40 + BPF_JA = 0 + BPF_JEQ = 0x10 + BPF_JGE = 0x30 + BPF_JGT = 0x20 + BPF_JMP = 0x5 + BPF_JSET = 0x40 + BPF_K = 0 + BPF_LD = 0 + BPF_LDX = 0x1 + BPF_LEN = 0x80 + BPF_LSH = 0x60 + BPF_MAJOR_VERSION = 0x1 + BPF_MAXBUFSIZE = 0x80000 + BPF_MAXINSNS = 0x200 + BPF_MEM = 0x60 + BPF_MEMWORDS = 0x10 + BPF_MINBUFSIZE = 0x20 + BPF_MINOR_VERSION = 0x1 + BPF_MISC = 0x7 + BPF_MSH = 0xa0 + BPF_MUL = 0x20 + BPF_NEG = 0x80 + BPF_OR = 0x40 + BPF_RELEASE = 0x30bb6 + BPF_RET = 0x6 + BPF_RSH = 0x70 + BPF_ST = 0x2 + BPF_STX = 0x3 + BPF_SUB = 0x10 + BPF_TAX = 0 + BPF_TXA = 0x80 + BPF_W = 0 + BPF_X = 0x8 + CTL_MAXNAME = 0x18 + CTL_NET = 0x4 + DLT_A429 = 0xb8 + DLT_A653_ICM = 0xb9 + DLT_AIRONET_HEADER = 0x78 + DLT_APPLE_IP_OVER_IEEE1394 = 0x8a + DLT_ARCNET = 0x7 + DLT_ARCNET_LINUX = 0x81 + DLT_ATM_CLIP = 0x13 + DLT_ATM_RFC1483 = 0xb + DLT_AURORA = 0x7e + DLT_AX25 = 0x3 + DLT_AX25_KISS = 0xca + DLT_BACNET_MS_TP = 0xa5 + DLT_BLUETOOTH_HCI_H4 = 0xbb + DLT_BLUETOOTH_HCI_H4_WITH_PHDR = 0xc9 + DLT_CAN20B = 0xbe + DLT_CHAOS = 0x5 + DLT_CHDLC = 0x68 + DLT_CISCO_IOS = 0x76 + DLT_C_HDLC = 0x68 + DLT_C_HDLC_WITH_DIR = 0xcd + DLT_DOCSIS = 0x8f + DLT_ECONET = 0x73 + DLT_EN10MB = 0x1 + DLT_EN3MB = 0x2 + DLT_ENC = 0x6d + DLT_ERF = 0xc5 + DLT_ERF_ETH = 0xaf + DLT_ERF_POS = 0xb0 + DLT_FDDI = 0xa + DLT_FLEXRAY = 0xd2 + DLT_FRELAY = 0x6b + DLT_FRELAY_WITH_DIR = 0xce + DLT_GCOM_SERIAL = 0xad + DLT_GCOM_T1E1 = 0xac + DLT_GPF_F = 0xab + DLT_GPF_T = 0xaa + DLT_GPRS_LLC = 0xa9 + DLT_HHDLC = 0x79 + DLT_IBM_SN = 0x92 + DLT_IBM_SP = 0x91 + DLT_IEEE802 = 0x6 + DLT_IEEE802_11 = 0x69 + DLT_IEEE802_11_RADIO = 0x7f + DLT_IEEE802_11_RADIO_AVS = 0xa3 + DLT_IEEE802_15_4 = 0xc3 + DLT_IEEE802_15_4_LINUX = 0xbf + DLT_IEEE802_15_4_NONASK_PHY = 0xd7 + DLT_IEEE802_16_MAC_CPS = 0xbc + DLT_IEEE802_16_MAC_CPS_RADIO = 0xc1 + DLT_IPFILTER = 0x74 + DLT_IPMB = 0xc7 + DLT_IPMB_LINUX = 0xd1 + DLT_IP_OVER_FC = 0x7a + DLT_JUNIPER_ATM1 = 0x89 + DLT_JUNIPER_ATM2 = 0x87 + DLT_JUNIPER_CHDLC = 0xb5 + DLT_JUNIPER_ES = 0x84 + DLT_JUNIPER_ETHER = 0xb2 + DLT_JUNIPER_FRELAY = 0xb4 + DLT_JUNIPER_GGSN = 0x85 + DLT_JUNIPER_ISM = 0xc2 + DLT_JUNIPER_MFR = 0x86 + DLT_JUNIPER_MLFR = 0x83 + DLT_JUNIPER_MLPPP = 0x82 + DLT_JUNIPER_MONITOR = 0xa4 + DLT_JUNIPER_PIC_PEER = 0xae + DLT_JUNIPER_PPP = 0xb3 + DLT_JUNIPER_PPPOE = 0xa7 + DLT_JUNIPER_PPPOE_ATM = 0xa8 + DLT_JUNIPER_SERVICES = 0x88 + DLT_JUNIPER_ST = 0xc8 + DLT_JUNIPER_VP = 0xb7 + DLT_LAPB_WITH_DIR = 0xcf + DLT_LAPD = 0xcb + DLT_LIN = 0xd4 + DLT_LINUX_IRDA = 0x90 + DLT_LINUX_LAPD = 0xb1 + DLT_LINUX_PPP_WITHDIRECTION = 0xa6 + DLT_LINUX_SLL = 0x71 + DLT_LOOP = 0x6c + DLT_LTALK = 0x72 + DLT_MFR = 0xb6 + DLT_MOST = 0xd3 + DLT_MTP2 = 0x8c + DLT_MTP2_WITH_PHDR = 0x8b + DLT_MTP3 = 0x8d + DLT_NULL = 0 + DLT_PCI_EXP = 0x7d + DLT_PFLOG = 0x75 + DLT_PFSYNC = 0x79 + DLT_PPI = 0xc0 + DLT_PPP = 0x9 + DLT_PPP_BSDOS = 0x10 + DLT_PPP_ETHER = 0x33 + DLT_PPP_PPPD = 0xa6 + DLT_PPP_SERIAL = 0x32 + DLT_PPP_WITH_DIR = 0xcc + DLT_PPP_WITH_DIRECTION = 0xa6 + DLT_PRISM_HEADER = 0x77 + DLT_PRONET = 0x4 + DLT_RAIF1 = 0xc6 + DLT_RAW = 0xc + DLT_RIO = 0x7c + DLT_SCCP = 0x8e + DLT_SITA = 0xc4 + DLT_SLIP = 0x8 + DLT_SLIP_BSDOS = 0xf + DLT_SUNATM = 0x7b + DLT_SYMANTEC_FIREWALL = 0x63 + DLT_TZSP = 0x80 + DLT_USB = 0xba + DLT_USB_LINUX = 0xbd + DLT_USER0 = 0x93 + DLT_USER1 = 0x94 + DLT_USER10 = 0x9d + DLT_USER11 = 0x9e + DLT_USER12 = 0x9f + DLT_USER13 = 0xa0 + DLT_USER14 = 0xa1 + DLT_USER15 = 0xa2 + DLT_USER2 = 0x95 + DLT_USER3 = 0x96 + DLT_USER4 = 0x97 + DLT_USER5 = 0x98 + DLT_USER6 = 0x99 + DLT_USER7 = 0x9a + DLT_USER8 = 0x9b + DLT_USER9 = 0x9c + DLT_X2E_SERIAL = 0xd5 + DLT_X2E_XORAYA = 0xd6 + DT_BLK = 0x6 + DT_CHR = 0x2 + DT_DIR = 0x4 + DT_FIFO = 0x1 + DT_LNK = 0xa + DT_REG = 0x8 + DT_SOCK = 0xc + DT_UNKNOWN = 0 + DT_WHT = 0xe + E2BIG = 0x7 + EACCES = 0xd + EADDRINUSE = 0x30 + EADDRNOTAVAIL = 0x31 + EAFNOSUPPORT = 0x2f + EAGAIN = 0x23 + EALREADY = 0x25 + EAUTH = 0x50 + EBADF = 0x9 + EBADMSG = 0x59 + EBADRPC = 0x48 + EBUSY = 0x10 + ECANCELED = 0x55 + ECHILD = 0xa + ECONNABORTED = 0x35 + ECONNREFUSED = 0x3d + ECONNRESET = 0x36 + EDEADLK = 0xb + EDESTADDRREQ = 0x27 + EDOM = 0x21 + EDOOFUS = 0x58 + EDQUOT = 0x45 + EEXIST = 0x11 + EFAULT = 0xe + EFBIG = 0x1b + EFTYPE = 0x4f + EHOSTDOWN = 0x40 + EHOSTUNREACH = 0x41 + EIDRM = 0x52 + EILSEQ = 0x56 + EINPROGRESS = 0x24 + EINTR = 0x4 + EINVAL = 0x16 + EIO = 0x5 + EISCONN = 0x38 + EISDIR = 0x15 + ELAST = 0x5d + ELOOP = 0x3e + EMFILE = 0x18 + EMLINK = 0x1f + EMSGSIZE = 0x28 + EMULTIHOP = 0x5a + ENAMETOOLONG = 0x3f + ENEEDAUTH = 0x51 + ENETDOWN = 0x32 + ENETRESET = 0x34 + ENETUNREACH = 0x33 + ENFILE = 0x17 + ENOATTR = 0x57 + ENOBUFS = 0x37 + ENODEV = 0x13 + ENOENT = 0x2 + ENOEXEC = 0x8 + ENOLCK = 0x4d + ENOLINK = 0x5b + ENOMEM = 0xc + ENOMSG = 0x53 + ENOPROTOOPT = 0x2a + ENOSPC = 0x1c + ENOSYS = 0x4e + ENOTBLK = 0xf + ENOTCAPABLE = 0x5d + ENOTCONN = 0x39 + ENOTDIR = 0x14 + ENOTEMPTY = 0x42 + ENOTSOCK = 0x26 + ENOTSUP = 0x2d + ENOTTY = 0x19 + ENXIO = 0x6 + EOPNOTSUPP = 0x2d + EOVERFLOW = 0x54 + EPERM = 0x1 + EPFNOSUPPORT = 0x2e + EPIPE = 0x20 + EPROCLIM = 0x43 + EPROCUNAVAIL = 0x4c + EPROGMISMATCH = 0x4b + EPROGUNAVAIL = 0x4a + EPROTO = 0x5c + EPROTONOSUPPORT = 0x2b + EPROTOTYPE = 0x29 + ERANGE = 0x22 + EREMOTE = 0x47 + EROFS = 0x1e + ERPCMISMATCH = 0x49 + ESHUTDOWN = 0x3a + ESOCKTNOSUPPORT = 0x2c + ESPIPE = 0x1d + ESRCH = 0x3 + ESTALE = 0x46 + ETIMEDOUT = 0x3c + ETOOMANYREFS = 0x3b + ETXTBSY = 0x1a + EUSERS = 0x44 + EVFILT_AIO = -0x3 + EVFILT_FS = -0x9 + EVFILT_LIO = -0xa + EVFILT_PROC = -0x5 + EVFILT_READ = -0x1 + EVFILT_SIGNAL = -0x6 + EVFILT_SYSCOUNT = 0xb + EVFILT_TIMER = -0x7 + EVFILT_USER = -0xb + EVFILT_VNODE = -0x4 + EVFILT_WRITE = -0x2 + EV_ADD = 0x1 + EV_CLEAR = 0x20 + EV_DELETE = 0x2 + EV_DISABLE = 0x8 + EV_DISPATCH = 0x80 + EV_ENABLE = 0x4 + EV_EOF = 0x8000 + EV_ERROR = 0x4000 + EV_FLAG1 = 0x2000 + EV_ONESHOT = 0x10 + EV_RECEIPT = 0x40 + EV_SYSFLAGS = 0xf000 + EWOULDBLOCK = 0x23 + EXDEV = 0x12 + FD_CLOEXEC = 0x1 + FD_SETSIZE = 0x400 + F_CANCEL = 0x5 + F_DUP2FD = 0xa + F_DUPFD = 0 + F_GETFD = 0x1 + F_GETFL = 0x3 + F_GETLK = 0xb + F_GETOWN = 0x5 + F_OGETLK = 0x7 + F_OSETLK = 0x8 + F_OSETLKW = 0x9 + F_RDAHEAD = 0x10 + F_RDLCK = 0x1 + F_READAHEAD = 0xf + F_SETFD = 0x2 + F_SETFL = 0x4 + F_SETLK = 0xc + F_SETLKW = 0xd + F_SETLK_REMOTE = 0xe + F_SETOWN = 0x6 + F_UNLCK = 0x2 + F_UNLCKSYS = 0x4 + F_WRLCK = 0x3 + IFF_ALLMULTI = 0x200 + IFF_ALTPHYS = 0x4000 + IFF_BROADCAST = 0x2 + IFF_CANTCHANGE = 0x208f72 + IFF_DEBUG = 0x4 + IFF_DRV_OACTIVE = 0x400 + IFF_DRV_RUNNING = 0x40 + IFF_DYING = 0x200000 + IFF_LINK0 = 0x1000 + IFF_LINK1 = 0x2000 + IFF_LINK2 = 0x4000 + IFF_LOOPBACK = 0x8 + IFF_MONITOR = 0x40000 + IFF_MULTICAST = 0x8000 + IFF_NOARP = 0x80 + IFF_OACTIVE = 0x400 + IFF_POINTOPOINT = 0x10 + IFF_PPROMISC = 0x20000 + IFF_PROMISC = 0x100 + IFF_RENAMING = 0x400000 + IFF_RUNNING = 0x40 + IFF_SIMPLEX = 0x800 + IFF_SMART = 0x20 + IFF_STATICARP = 0x80000 + IFF_UP = 0x1 + IFNAMSIZ = 0x10 + IFT_1822 = 0x2 + IFT_A12MPPSWITCH = 0x82 + IFT_AAL2 = 0xbb + IFT_AAL5 = 0x31 + IFT_ADSL = 0x5e + IFT_AFLANE8023 = 0x3b + IFT_AFLANE8025 = 0x3c + IFT_ARAP = 0x58 + IFT_ARCNET = 0x23 + IFT_ARCNETPLUS = 0x24 + IFT_ASYNC = 0x54 + IFT_ATM = 0x25 + IFT_ATMDXI = 0x69 + IFT_ATMFUNI = 0x6a + IFT_ATMIMA = 0x6b + IFT_ATMLOGICAL = 0x50 + IFT_ATMRADIO = 0xbd + IFT_ATMSUBINTERFACE = 0x86 + IFT_ATMVCIENDPT = 0xc2 + IFT_ATMVIRTUAL = 0x95 + IFT_BGPPOLICYACCOUNTING = 0xa2 + IFT_BRIDGE = 0xd1 + IFT_BSC = 0x53 + IFT_CARP = 0xf8 + IFT_CCTEMUL = 0x3d + IFT_CEPT = 0x13 + IFT_CES = 0x85 + IFT_CHANNEL = 0x46 + IFT_CNR = 0x55 + IFT_COFFEE = 0x84 + IFT_COMPOSITELINK = 0x9b + IFT_DCN = 0x8d + IFT_DIGITALPOWERLINE = 0x8a + IFT_DIGITALWRAPPEROVERHEADCHANNEL = 0xba + IFT_DLSW = 0x4a + IFT_DOCSCABLEDOWNSTREAM = 0x80 + IFT_DOCSCABLEMACLAYER = 0x7f + IFT_DOCSCABLEUPSTREAM = 0x81 + IFT_DS0 = 0x51 + IFT_DS0BUNDLE = 0x52 + IFT_DS1FDL = 0xaa + IFT_DS3 = 0x1e + IFT_DTM = 0x8c + IFT_DVBASILN = 0xac + IFT_DVBASIOUT = 0xad + IFT_DVBRCCDOWNSTREAM = 0x93 + IFT_DVBRCCMACLAYER = 0x92 + IFT_DVBRCCUPSTREAM = 0x94 + IFT_ENC = 0xf4 + IFT_EON = 0x19 + IFT_EPLRS = 0x57 + IFT_ESCON = 0x49 + IFT_ETHER = 0x6 + IFT_FAITH = 0xf2 + IFT_FAST = 0x7d + IFT_FASTETHER = 0x3e + IFT_FASTETHERFX = 0x45 + IFT_FDDI = 0xf + IFT_FIBRECHANNEL = 0x38 + IFT_FRAMERELAYINTERCONNECT = 0x3a + IFT_FRAMERELAYMPI = 0x5c + IFT_FRDLCIENDPT = 0xc1 + IFT_FRELAY = 0x20 + IFT_FRELAYDCE = 0x2c + IFT_FRF16MFRBUNDLE = 0xa3 + IFT_FRFORWARD = 0x9e + IFT_G703AT2MB = 0x43 + IFT_G703AT64K = 0x42 + IFT_GIF = 0xf0 + IFT_GIGABITETHERNET = 0x75 + IFT_GR303IDT = 0xb2 + IFT_GR303RDT = 0xb1 + IFT_H323GATEKEEPER = 0xa4 + IFT_H323PROXY = 0xa5 + IFT_HDH1822 = 0x3 + IFT_HDLC = 0x76 + IFT_HDSL2 = 0xa8 + IFT_HIPERLAN2 = 0xb7 + IFT_HIPPI = 0x2f + IFT_HIPPIINTERFACE = 0x39 + IFT_HOSTPAD = 0x5a + IFT_HSSI = 0x2e + IFT_HY = 0xe + IFT_IBM370PARCHAN = 0x48 + IFT_IDSL = 0x9a + IFT_IEEE1394 = 0x90 + IFT_IEEE80211 = 0x47 + IFT_IEEE80212 = 0x37 + IFT_IEEE8023ADLAG = 0xa1 + IFT_IFGSN = 0x91 + IFT_IMT = 0xbe + IFT_INTERLEAVE = 0x7c + IFT_IP = 0x7e + IFT_IPFORWARD = 0x8e + IFT_IPOVERATM = 0x72 + IFT_IPOVERCDLC = 0x6d + IFT_IPOVERCLAW = 0x6e + IFT_IPSWITCH = 0x4e + IFT_IPXIP = 0xf9 + IFT_ISDN = 0x3f + IFT_ISDNBASIC = 0x14 + IFT_ISDNPRIMARY = 0x15 + IFT_ISDNS = 0x4b + IFT_ISDNU = 0x4c + IFT_ISO88022LLC = 0x29 + IFT_ISO88023 = 0x7 + IFT_ISO88024 = 0x8 + IFT_ISO88025 = 0x9 + IFT_ISO88025CRFPINT = 0x62 + IFT_ISO88025DTR = 0x56 + IFT_ISO88025FIBER = 0x73 + IFT_ISO88026 = 0xa + IFT_ISUP = 0xb3 + IFT_L2VLAN = 0x87 + IFT_L3IPVLAN = 0x88 + IFT_L3IPXVLAN = 0x89 + IFT_LAPB = 0x10 + IFT_LAPD = 0x4d + IFT_LAPF = 0x77 + IFT_LOCALTALK = 0x2a + IFT_LOOP = 0x18 + IFT_MEDIAMAILOVERIP = 0x8b + IFT_MFSIGLINK = 0xa7 + IFT_MIOX25 = 0x26 + IFT_MODEM = 0x30 + IFT_MPC = 0x71 + IFT_MPLS = 0xa6 + IFT_MPLSTUNNEL = 0x96 + IFT_MSDSL = 0x8f + IFT_MVL = 0xbf + IFT_MYRINET = 0x63 + IFT_NFAS = 0xaf + IFT_NSIP = 0x1b + IFT_OPTICALCHANNEL = 0xc3 + IFT_OPTICALTRANSPORT = 0xc4 + IFT_OTHER = 0x1 + IFT_P10 = 0xc + IFT_P80 = 0xd + IFT_PARA = 0x22 + IFT_PFLOG = 0xf6 + IFT_PFSYNC = 0xf7 + IFT_PLC = 0xae + IFT_POS = 0xab + IFT_PPP = 0x17 + IFT_PPPMULTILINKBUNDLE = 0x6c + IFT_PROPBWAP2MP = 0xb8 + IFT_PROPCNLS = 0x59 + IFT_PROPDOCSWIRELESSDOWNSTREAM = 0xb5 + IFT_PROPDOCSWIRELESSMACLAYER = 0xb4 + IFT_PROPDOCSWIRELESSUPSTREAM = 0xb6 + IFT_PROPMUX = 0x36 + IFT_PROPVIRTUAL = 0x35 + IFT_PROPWIRELESSP2P = 0x9d + IFT_PTPSERIAL = 0x16 + IFT_PVC = 0xf1 + IFT_QLLC = 0x44 + IFT_RADIOMAC = 0xbc + IFT_RADSL = 0x5f + IFT_REACHDSL = 0xc0 + IFT_RFC1483 = 0x9f + IFT_RS232 = 0x21 + IFT_RSRB = 0x4f + IFT_SDLC = 0x11 + IFT_SDSL = 0x60 + IFT_SHDSL = 0xa9 + IFT_SIP = 0x1f + IFT_SLIP = 0x1c + IFT_SMDSDXI = 0x2b + IFT_SMDSICIP = 0x34 + IFT_SONET = 0x27 + IFT_SONETOVERHEADCHANNEL = 0xb9 + IFT_SONETPATH = 0x32 + IFT_SONETVT = 0x33 + IFT_SRP = 0x97 + IFT_SS7SIGLINK = 0x9c + IFT_STACKTOSTACK = 0x6f + IFT_STARLAN = 0xb + IFT_STF = 0xd7 + IFT_T1 = 0x12 + IFT_TDLC = 0x74 + IFT_TERMPAD = 0x5b + IFT_TR008 = 0xb0 + IFT_TRANSPHDLC = 0x7b + IFT_TUNNEL = 0x83 + IFT_ULTRA = 0x1d + IFT_USB = 0xa0 + IFT_V11 = 0x40 + IFT_V35 = 0x2d + IFT_V36 = 0x41 + IFT_V37 = 0x78 + IFT_VDSL = 0x61 + IFT_VIRTUALIPADDRESS = 0x70 + IFT_VOICEEM = 0x64 + IFT_VOICEENCAP = 0x67 + IFT_VOICEFXO = 0x65 + IFT_VOICEFXS = 0x66 + IFT_VOICEOVERATM = 0x98 + IFT_VOICEOVERFRAMERELAY = 0x99 + IFT_VOICEOVERIP = 0x68 + IFT_X213 = 0x5d + IFT_X25 = 0x5 + IFT_X25DDN = 0x4 + IFT_X25HUNTGROUP = 0x7a + IFT_X25MLP = 0x79 + IFT_X25PLE = 0x28 + IFT_XETHER = 0x1a + IN_CLASSA_HOST = 0xffffff + IN_CLASSA_MAX = 0x80 + IN_CLASSA_NET = 0xff000000 + IN_CLASSA_NSHIFT = 0x18 + IN_CLASSB_HOST = 0xffff + IN_CLASSB_MAX = 0x10000 + IN_CLASSB_NET = 0xffff0000 + IN_CLASSB_NSHIFT = 0x10 + IN_CLASSC_HOST = 0xff + IN_CLASSC_NET = 0xffffff00 + IN_CLASSC_NSHIFT = 0x8 + IN_CLASSD_HOST = 0xfffffff + IN_CLASSD_NET = 0xf0000000 + IN_CLASSD_NSHIFT = 0x1c + IN_LOOPBACKNET = 0x7f + IPPROTO_3PC = 0x22 + IPPROTO_ADFS = 0x44 + IPPROTO_AH = 0x33 + IPPROTO_AHIP = 0x3d + IPPROTO_APES = 0x63 + IPPROTO_ARGUS = 0xd + IPPROTO_AX25 = 0x5d + IPPROTO_BHA = 0x31 + IPPROTO_BLT = 0x1e + IPPROTO_BRSATMON = 0x4c + IPPROTO_CARP = 0x70 + IPPROTO_CFTP = 0x3e + IPPROTO_CHAOS = 0x10 + IPPROTO_CMTP = 0x26 + IPPROTO_CPHB = 0x49 + IPPROTO_CPNX = 0x48 + IPPROTO_DDP = 0x25 + IPPROTO_DGP = 0x56 + IPPROTO_DIVERT = 0x102 + IPPROTO_DONE = 0x101 + IPPROTO_DSTOPTS = 0x3c + IPPROTO_EGP = 0x8 + IPPROTO_EMCON = 0xe + IPPROTO_ENCAP = 0x62 + IPPROTO_EON = 0x50 + IPPROTO_ESP = 0x32 + IPPROTO_ETHERIP = 0x61 + IPPROTO_FRAGMENT = 0x2c + IPPROTO_GGP = 0x3 + IPPROTO_GMTP = 0x64 + IPPROTO_GRE = 0x2f + IPPROTO_HELLO = 0x3f + IPPROTO_HMP = 0x14 + IPPROTO_HOPOPTS = 0 + IPPROTO_ICMP = 0x1 + IPPROTO_ICMPV6 = 0x3a + IPPROTO_IDP = 0x16 + IPPROTO_IDPR = 0x23 + IPPROTO_IDRP = 0x2d + IPPROTO_IGMP = 0x2 + IPPROTO_IGP = 0x55 + IPPROTO_IGRP = 0x58 + IPPROTO_IL = 0x28 + IPPROTO_INLSP = 0x34 + IPPROTO_INP = 0x20 + IPPROTO_IP = 0 + IPPROTO_IPCOMP = 0x6c + IPPROTO_IPCV = 0x47 + IPPROTO_IPEIP = 0x5e + IPPROTO_IPIP = 0x4 + IPPROTO_IPPC = 0x43 + IPPROTO_IPV4 = 0x4 + IPPROTO_IPV6 = 0x29 + IPPROTO_IRTP = 0x1c + IPPROTO_KRYPTOLAN = 0x41 + IPPROTO_LARP = 0x5b + IPPROTO_LEAF1 = 0x19 + IPPROTO_LEAF2 = 0x1a + IPPROTO_MAX = 0x100 + IPPROTO_MAXID = 0x34 + IPPROTO_MEAS = 0x13 + IPPROTO_MHRP = 0x30 + IPPROTO_MICP = 0x5f + IPPROTO_MOBILE = 0x37 + IPPROTO_MTP = 0x5c + IPPROTO_MUX = 0x12 + IPPROTO_ND = 0x4d + IPPROTO_NHRP = 0x36 + IPPROTO_NONE = 0x3b + IPPROTO_NSP = 0x1f + IPPROTO_NVPII = 0xb + IPPROTO_OLD_DIVERT = 0xfe + IPPROTO_OSPFIGP = 0x59 + IPPROTO_PFSYNC = 0xf0 + IPPROTO_PGM = 0x71 + IPPROTO_PIGP = 0x9 + IPPROTO_PIM = 0x67 + IPPROTO_PRM = 0x15 + IPPROTO_PUP = 0xc + IPPROTO_PVP = 0x4b + IPPROTO_RAW = 0xff + IPPROTO_RCCMON = 0xa + IPPROTO_RDP = 0x1b + IPPROTO_ROUTING = 0x2b + IPPROTO_RSVP = 0x2e + IPPROTO_RVD = 0x42 + IPPROTO_SATEXPAK = 0x40 + IPPROTO_SATMON = 0x45 + IPPROTO_SCCSP = 0x60 + IPPROTO_SCTP = 0x84 + IPPROTO_SDRP = 0x2a + IPPROTO_SEP = 0x21 + IPPROTO_SKIP = 0x39 + IPPROTO_SPACER = 0x7fff + IPPROTO_SRPC = 0x5a + IPPROTO_ST = 0x7 + IPPROTO_SVMTP = 0x52 + IPPROTO_SWIPE = 0x35 + IPPROTO_TCF = 0x57 + IPPROTO_TCP = 0x6 + IPPROTO_TLSP = 0x38 + IPPROTO_TP = 0x1d + IPPROTO_TPXX = 0x27 + IPPROTO_TRUNK1 = 0x17 + IPPROTO_TRUNK2 = 0x18 + IPPROTO_TTP = 0x54 + IPPROTO_UDP = 0x11 + IPPROTO_VINES = 0x53 + IPPROTO_VISA = 0x46 + IPPROTO_VMTP = 0x51 + IPPROTO_WBEXPAK = 0x4f + IPPROTO_WBMON = 0x4e + IPPROTO_WSN = 0x4a + IPPROTO_XNET = 0xf + IPPROTO_XTP = 0x24 + IPV6_AUTOFLOWLABEL = 0x3b + IPV6_BINDANY = 0x40 + IPV6_BINDV6ONLY = 0x1b + IPV6_CHECKSUM = 0x1a + IPV6_DEFAULT_MULTICAST_HOPS = 0x1 + IPV6_DEFAULT_MULTICAST_LOOP = 0x1 + IPV6_DEFHLIM = 0x40 + IPV6_DONTFRAG = 0x3e + IPV6_DSTOPTS = 0x32 + IPV6_FAITH = 0x1d + IPV6_FLOWINFO_MASK = 0xffffff0f + IPV6_FLOWLABEL_MASK = 0xffff0f00 + IPV6_FRAGTTL = 0x78 + IPV6_FW_ADD = 0x1e + IPV6_FW_DEL = 0x1f + IPV6_FW_FLUSH = 0x20 + IPV6_FW_GET = 0x22 + IPV6_FW_ZERO = 0x21 + IPV6_HLIMDEC = 0x1 + IPV6_HOPLIMIT = 0x2f + IPV6_HOPOPTS = 0x31 + IPV6_IPSEC_POLICY = 0x1c + IPV6_JOIN_GROUP = 0xc + IPV6_LEAVE_GROUP = 0xd + IPV6_MAXHLIM = 0xff + IPV6_MAXOPTHDR = 0x800 + IPV6_MAXPACKET = 0xffff + IPV6_MAX_GROUP_SRC_FILTER = 0x200 + IPV6_MAX_MEMBERSHIPS = 0xfff + IPV6_MAX_SOCK_SRC_FILTER = 0x80 + IPV6_MIN_MEMBERSHIPS = 0x1f + IPV6_MMTU = 0x500 + IPV6_MSFILTER = 0x4a + IPV6_MULTICAST_HOPS = 0xa + IPV6_MULTICAST_IF = 0x9 + IPV6_MULTICAST_LOOP = 0xb + IPV6_NEXTHOP = 0x30 + IPV6_PATHMTU = 0x2c + IPV6_PKTINFO = 0x2e + IPV6_PORTRANGE = 0xe + IPV6_PORTRANGE_DEFAULT = 0 + IPV6_PORTRANGE_HIGH = 0x1 + IPV6_PORTRANGE_LOW = 0x2 + IPV6_PREFER_TEMPADDR = 0x3f + IPV6_RECVDSTOPTS = 0x28 + IPV6_RECVHOPLIMIT = 0x25 + IPV6_RECVHOPOPTS = 0x27 + IPV6_RECVPATHMTU = 0x2b + IPV6_RECVPKTINFO = 0x24 + IPV6_RECVRTHDR = 0x26 + IPV6_RECVTCLASS = 0x39 + IPV6_RTHDR = 0x33 + IPV6_RTHDRDSTOPTS = 0x23 + IPV6_RTHDR_LOOSE = 0 + IPV6_RTHDR_STRICT = 0x1 + IPV6_RTHDR_TYPE_0 = 0 + IPV6_SOCKOPT_RESERVED1 = 0x3 + IPV6_TCLASS = 0x3d + IPV6_UNICAST_HOPS = 0x4 + IPV6_USE_MIN_MTU = 0x2a + IPV6_V6ONLY = 0x1b + IPV6_VERSION = 0x60 + IPV6_VERSION_MASK = 0xf0 + IP_ADD_MEMBERSHIP = 0xc + IP_ADD_SOURCE_MEMBERSHIP = 0x46 + IP_BINDANY = 0x18 + IP_BLOCK_SOURCE = 0x48 + IP_DEFAULT_MULTICAST_LOOP = 0x1 + IP_DEFAULT_MULTICAST_TTL = 0x1 + IP_DF = 0x4000 + IP_DONTFRAG = 0x43 + IP_DROP_MEMBERSHIP = 0xd + IP_DROP_SOURCE_MEMBERSHIP = 0x47 + IP_DUMMYNET3 = 0x31 + IP_DUMMYNET_CONFIGURE = 0x3c + IP_DUMMYNET_DEL = 0x3d + IP_DUMMYNET_FLUSH = 0x3e + IP_DUMMYNET_GET = 0x40 + IP_FAITH = 0x16 + IP_FW3 = 0x30 + IP_FW_ADD = 0x32 + IP_FW_DEL = 0x33 + IP_FW_FLUSH = 0x34 + IP_FW_GET = 0x36 + IP_FW_NAT_CFG = 0x38 + IP_FW_NAT_DEL = 0x39 + IP_FW_NAT_GET_CONFIG = 0x3a + IP_FW_NAT_GET_LOG = 0x3b + IP_FW_RESETLOG = 0x37 + IP_FW_TABLE_ADD = 0x28 + IP_FW_TABLE_DEL = 0x29 + IP_FW_TABLE_FLUSH = 0x2a + IP_FW_TABLE_GETSIZE = 0x2b + IP_FW_TABLE_LIST = 0x2c + IP_FW_ZERO = 0x35 + IP_HDRINCL = 0x2 + IP_IPSEC_POLICY = 0x15 + IP_MAXPACKET = 0xffff + IP_MAX_GROUP_SRC_FILTER = 0x200 + IP_MAX_MEMBERSHIPS = 0xfff + IP_MAX_SOCK_MUTE_FILTER = 0x80 + IP_MAX_SOCK_SRC_FILTER = 0x80 + IP_MAX_SOURCE_FILTER = 0x400 + IP_MF = 0x2000 + IP_MINTTL = 0x42 + IP_MIN_MEMBERSHIPS = 0x1f + IP_MSFILTER = 0x4a + IP_MSS = 0x240 + IP_MULTICAST_IF = 0x9 + IP_MULTICAST_LOOP = 0xb + IP_MULTICAST_TTL = 0xa + IP_MULTICAST_VIF = 0xe + IP_OFFMASK = 0x1fff + IP_ONESBCAST = 0x17 + IP_OPTIONS = 0x1 + IP_PORTRANGE = 0x13 + IP_PORTRANGE_DEFAULT = 0 + IP_PORTRANGE_HIGH = 0x1 + IP_PORTRANGE_LOW = 0x2 + IP_RECVDSTADDR = 0x7 + IP_RECVIF = 0x14 + IP_RECVOPTS = 0x5 + IP_RECVRETOPTS = 0x6 + IP_RECVTTL = 0x41 + IP_RETOPTS = 0x8 + IP_RF = 0x8000 + IP_RSVP_OFF = 0x10 + IP_RSVP_ON = 0xf + IP_RSVP_VIF_OFF = 0x12 + IP_RSVP_VIF_ON = 0x11 + IP_SENDSRCADDR = 0x7 + IP_TOS = 0x3 + IP_TTL = 0x4 + IP_UNBLOCK_SOURCE = 0x49 + MSG_COMPAT = 0x8000 + MSG_CTRUNC = 0x20 + MSG_DONTROUTE = 0x4 + MSG_DONTWAIT = 0x80 + MSG_EOF = 0x100 + MSG_EOR = 0x8 + MSG_NBIO = 0x4000 + MSG_NOSIGNAL = 0x20000 + MSG_NOTIFICATION = 0x2000 + MSG_OOB = 0x1 + MSG_PEEK = 0x2 + MSG_TRUNC = 0x10 + MSG_WAITALL = 0x40 + NET_RT_DUMP = 0x1 + NET_RT_FLAGS = 0x2 + NET_RT_IFLIST = 0x3 + NET_RT_IFMALIST = 0x4 + NET_RT_MAXID = 0x5 + O_ACCMODE = 0x3 + O_APPEND = 0x8 + O_ASYNC = 0x40 + O_CREAT = 0x200 + O_DIRECT = 0x10000 + O_DIRECTORY = 0x20000 + O_EXCL = 0x800 + O_EXEC = 0x40000 + O_EXLOCK = 0x20 + O_FSYNC = 0x80 + O_NDELAY = 0x4 + O_NOCTTY = 0x8000 + O_NOFOLLOW = 0x100 + O_NONBLOCK = 0x4 + O_RDONLY = 0 + O_RDWR = 0x2 + O_SHLOCK = 0x10 + O_SYNC = 0x80 + O_TRUNC = 0x400 + O_TTY_INIT = 0x80000 + O_WRONLY = 0x1 + RTAX_AUTHOR = 0x6 + RTAX_BRD = 0x7 + RTAX_DST = 0 + RTAX_GATEWAY = 0x1 + RTAX_GENMASK = 0x3 + RTAX_IFA = 0x5 + RTAX_IFP = 0x4 + RTAX_MAX = 0x8 + RTAX_NETMASK = 0x2 + RTA_AUTHOR = 0x40 + RTA_BRD = 0x80 + RTA_DST = 0x1 + RTA_GATEWAY = 0x2 + RTA_GENMASK = 0x8 + RTA_IFA = 0x20 + RTA_IFP = 0x10 + RTA_NETMASK = 0x4 + RTF_BLACKHOLE = 0x1000 + RTF_BROADCAST = 0x400000 + RTF_DONE = 0x40 + RTF_DYNAMIC = 0x10 + RTF_FMASK = 0x1004d808 + RTF_GATEWAY = 0x2 + RTF_HOST = 0x4 + RTF_LLDATA = 0x400 + RTF_LLINFO = 0x400 + RTF_LOCAL = 0x200000 + RTF_MODIFIED = 0x20 + RTF_MULTICAST = 0x800000 + RTF_PINNED = 0x100000 + RTF_PRCLONING = 0x10000 + RTF_PROTO1 = 0x8000 + RTF_PROTO2 = 0x4000 + RTF_PROTO3 = 0x40000 + RTF_REJECT = 0x8 + RTF_RNH_LOCKED = 0x40000000 + RTF_STATIC = 0x800 + RTF_STICKY = 0x10000000 + RTF_UP = 0x1 + RTF_XRESOLVE = 0x200 + RTM_ADD = 0x1 + RTM_CHANGE = 0x3 + RTM_DELADDR = 0xd + RTM_DELETE = 0x2 + RTM_DELMADDR = 0x10 + RTM_GET = 0x4 + RTM_IEEE80211 = 0x12 + RTM_IFANNOUNCE = 0x11 + RTM_IFINFO = 0xe + RTM_LOCK = 0x8 + RTM_LOSING = 0x5 + RTM_MISS = 0x7 + RTM_NEWADDR = 0xc + RTM_NEWMADDR = 0xf + RTM_OLDADD = 0x9 + RTM_OLDDEL = 0xa + RTM_REDIRECT = 0x6 + RTM_RESOLVE = 0xb + RTM_RTTUNIT = 0xf4240 + RTM_VERSION = 0x5 + RTV_EXPIRE = 0x4 + RTV_HOPCOUNT = 0x2 + RTV_MTU = 0x1 + RTV_RPIPE = 0x8 + RTV_RTT = 0x40 + RTV_RTTVAR = 0x80 + RTV_SPIPE = 0x10 + RTV_SSTHRESH = 0x20 + RTV_WEIGHT = 0x100 + SCM_BINTIME = 0x4 + SCM_CREDS = 0x3 + SCM_RIGHTS = 0x1 + SCM_TIMESTAMP = 0x2 + SHUT_RD = 0 + SHUT_RDWR = 0x2 + SHUT_WR = 0x1 + SIGABRT = 0x6 + SIGALRM = 0xe + SIGBUS = 0xa + SIGCHLD = 0x14 + SIGCONT = 0x13 + SIGEMT = 0x7 + SIGFPE = 0x8 + SIGHUP = 0x1 + SIGILL = 0x4 + SIGINFO = 0x1d + SIGINT = 0x2 + SIGIO = 0x17 + SIGIOT = 0x6 + SIGKILL = 0x9 + SIGLWP = 0x20 + SIGPIPE = 0xd + SIGPROF = 0x1b + SIGQUIT = 0x3 + SIGSEGV = 0xb + SIGSTOP = 0x11 + SIGSYS = 0xc + SIGTERM = 0xf + SIGTHR = 0x20 + SIGTRAP = 0x5 + SIGTSTP = 0x12 + SIGTTIN = 0x15 + SIGTTOU = 0x16 + SIGURG = 0x10 + SIGUSR1 = 0x1e + SIGUSR2 = 0x1f + SIGVTALRM = 0x1a + SIGWINCH = 0x1c + SIGXCPU = 0x18 + SIGXFSZ = 0x19 + SIOCADDMULTI = 0x80206931 + SIOCADDRT = 0x8040720a + SIOCAIFADDR = 0x8040691a + SIOCAIFGROUP = 0x80286987 + SIOCALIFADDR = 0x8118691b + SIOCATMARK = 0x40047307 + SIOCDELMULTI = 0x80206932 + SIOCDELRT = 0x8040720b + SIOCDIFADDR = 0x80206919 + SIOCDIFGROUP = 0x80286989 + SIOCDIFPHYADDR = 0x80206949 + SIOCDLIFADDR = 0x8118691d + SIOCGDRVSPEC = 0xc028697b + SIOCGETSGCNT = 0xc0207210 + SIOCGETVIFCNT = 0xc028720f + SIOCGHIWAT = 0x40047301 + SIOCGIFADDR = 0xc0206921 + SIOCGIFBRDADDR = 0xc0206923 + SIOCGIFCAP = 0xc020691f + SIOCGIFCONF = 0xc0106924 + SIOCGIFDESCR = 0xc020692a + SIOCGIFDSTADDR = 0xc0206922 + SIOCGIFFLAGS = 0xc0206911 + SIOCGIFGENERIC = 0xc020693a + SIOCGIFGMEMB = 0xc028698a + SIOCGIFGROUP = 0xc0286988 + SIOCGIFINDEX = 0xc0206920 + SIOCGIFMAC = 0xc0206926 + SIOCGIFMEDIA = 0xc0306938 + SIOCGIFMETRIC = 0xc0206917 + SIOCGIFMTU = 0xc0206933 + SIOCGIFNETMASK = 0xc0206925 + SIOCGIFPDSTADDR = 0xc0206948 + SIOCGIFPHYS = 0xc0206935 + SIOCGIFPSRCADDR = 0xc0206947 + SIOCGIFSTATUS = 0xc331693b + SIOCGLIFADDR = 0xc118691c + SIOCGLIFPHYADDR = 0xc118694b + SIOCGLOWAT = 0x40047303 + SIOCGPGRP = 0x40047309 + SIOCGPRIVATE_0 = 0xc0206950 + SIOCGPRIVATE_1 = 0xc0206951 + SIOCIFCREATE = 0xc020697a + SIOCIFCREATE2 = 0xc020697c + SIOCIFDESTROY = 0x80206979 + SIOCIFGCLONERS = 0xc0106978 + SIOCSDRVSPEC = 0x8028697b + SIOCSHIWAT = 0x80047300 + SIOCSIFADDR = 0x8020690c + SIOCSIFBRDADDR = 0x80206913 + SIOCSIFCAP = 0x8020691e + SIOCSIFDESCR = 0x80206929 + SIOCSIFDSTADDR = 0x8020690e + SIOCSIFFLAGS = 0x80206910 + SIOCSIFGENERIC = 0x80206939 + SIOCSIFLLADDR = 0x8020693c + SIOCSIFMAC = 0x80206927 + SIOCSIFMEDIA = 0xc0206937 + SIOCSIFMETRIC = 0x80206918 + SIOCSIFMTU = 0x80206934 + SIOCSIFNAME = 0x80206928 + SIOCSIFNETMASK = 0x80206916 + SIOCSIFPHYADDR = 0x80406946 + SIOCSIFPHYS = 0x80206936 + SIOCSIFRVNET = 0xc020695b + SIOCSIFVNET = 0xc020695a + SIOCSLIFPHYADDR = 0x8118694a + SIOCSLOWAT = 0x80047302 + SIOCSPGRP = 0x80047308 + SOCK_DGRAM = 0x2 + SOCK_MAXADDRLEN = 0xff + SOCK_RAW = 0x3 + SOCK_RDM = 0x4 + SOCK_SEQPACKET = 0x5 + SOCK_STREAM = 0x1 + SOL_SOCKET = 0xffff + SOMAXCONN = 0x80 + SO_ACCEPTCONN = 0x2 + SO_ACCEPTFILTER = 0x1000 + SO_BINTIME = 0x2000 + SO_BROADCAST = 0x20 + SO_DEBUG = 0x1 + SO_DONTROUTE = 0x10 + SO_ERROR = 0x1007 + SO_KEEPALIVE = 0x8 + SO_LABEL = 0x1009 + SO_LINGER = 0x80 + SO_LISTENINCQLEN = 0x1013 + SO_LISTENQLEN = 0x1012 + SO_LISTENQLIMIT = 0x1011 + SO_NOSIGPIPE = 0x800 + SO_NO_DDP = 0x8000 + SO_NO_OFFLOAD = 0x4000 + SO_OOBINLINE = 0x100 + SO_PEERLABEL = 0x1010 + SO_RCVBUF = 0x1002 + SO_RCVLOWAT = 0x1004 + SO_RCVTIMEO = 0x1006 + SO_REUSEADDR = 0x4 + SO_REUSEPORT = 0x200 + SO_SETFIB = 0x1014 + SO_SNDBUF = 0x1001 + SO_SNDLOWAT = 0x1003 + SO_SNDTIMEO = 0x1005 + SO_TIMESTAMP = 0x400 + SO_TYPE = 0x1008 + SO_USELOOPBACK = 0x40 + TCP_CA_NAME_MAX = 0x10 + TCP_CONGESTION = 0x40 + TCP_INFO = 0x20 + TCP_MAXBURST = 0x4 + TCP_MAXHLEN = 0x3c + TCP_MAXOLEN = 0x28 + TCP_MAXSEG = 0x2 + TCP_MAXWIN = 0xffff + TCP_MAX_SACK = 0x4 + TCP_MAX_WINSHIFT = 0xe + TCP_MD5SIG = 0x10 + TCP_MINMSS = 0xd8 + TCP_MSS = 0x200 + TCP_NODELAY = 0x1 + TCP_NOOPT = 0x8 + TCP_NOPUSH = 0x4 + WCONTINUED = 0x4 + WCOREFLAG = 0x80 + WLINUXCLONE = 0x80000000 + WNOHANG = 0x1 + WNOWAIT = 0x8 + WSTOPPED = 0x2 + WUNTRACED = 0x2 ) // Types diff --git a/src/pkg/syscall/zerrors_linux_386.go b/src/pkg/syscall/zerrors_linux_386.go index 3fc7cc738..73caed44c 100644 --- a/src/pkg/syscall/zerrors_linux_386.go +++ b/src/pkg/syscall/zerrors_linux_386.go @@ -48,6 +48,64 @@ const ( AF_UNSPEC = 0 AF_WANPIPE = 0x19 AF_X25 = 0x9 + ARPHRD_ADAPT = 0x108 + ARPHRD_APPLETLK = 0x8 + ARPHRD_ARCNET = 0x7 + ARPHRD_ASH = 0x30d + ARPHRD_ATM = 0x13 + ARPHRD_AX25 = 0x3 + ARPHRD_BIF = 0x307 + ARPHRD_CHAOS = 0x5 + ARPHRD_CISCO = 0x201 + ARPHRD_CSLIP = 0x101 + ARPHRD_CSLIP6 = 0x103 + ARPHRD_DDCMP = 0x205 + ARPHRD_DLCI = 0xf + ARPHRD_ECONET = 0x30e + ARPHRD_EETHER = 0x2 + ARPHRD_ETHER = 0x1 + ARPHRD_EUI64 = 0x1b + ARPHRD_FCAL = 0x311 + ARPHRD_FCFABRIC = 0x313 + ARPHRD_FCPL = 0x312 + ARPHRD_FCPP = 0x310 + ARPHRD_FDDI = 0x306 + ARPHRD_FRAD = 0x302 + ARPHRD_HDLC = 0x201 + ARPHRD_HIPPI = 0x30c + ARPHRD_HWX25 = 0x110 + ARPHRD_IEEE1394 = 0x18 + ARPHRD_IEEE802 = 0x6 + ARPHRD_IEEE80211 = 0x321 + ARPHRD_IEEE80211_PRISM = 0x322 + ARPHRD_IEEE80211_RADIOTAP = 0x323 + ARPHRD_IEEE802154 = 0x324 + ARPHRD_IEEE802154_PHY = 0x325 + ARPHRD_IEEE802_TR = 0x320 + ARPHRD_INFINIBAND = 0x20 + ARPHRD_IPDDP = 0x309 + ARPHRD_IPGRE = 0x30a + ARPHRD_IRDA = 0x30f + ARPHRD_LAPB = 0x204 + ARPHRD_LOCALTLK = 0x305 + ARPHRD_LOOPBACK = 0x304 + ARPHRD_METRICOM = 0x17 + ARPHRD_NETROM = 0 + ARPHRD_NONE = 0xfffe + ARPHRD_PIMREG = 0x30b + ARPHRD_PPP = 0x200 + ARPHRD_PRONET = 0x4 + ARPHRD_RAWHDLC = 0x206 + ARPHRD_ROSE = 0x10e + ARPHRD_RSRVD = 0x104 + ARPHRD_SIT = 0x308 + ARPHRD_SKIP = 0x303 + ARPHRD_SLIP = 0x100 + ARPHRD_SLIP6 = 0x102 + ARPHRD_TUNNEL = 0x300 + ARPHRD_TUNNEL6 = 0x301 + ARPHRD_VOID = 0xffff + ARPHRD_X25 = 0x10f DT_BLK = 0x6 DT_CHR = 0x2 DT_DIR = 0x4 @@ -241,6 +299,16 @@ const ( F_ULOCK = 0 F_UNLCK = 0x2 F_WRLCK = 0x1 + IFA_F_DADFAILED = 0x8 + IFA_F_DEPRECATED = 0x20 + IFA_F_HOMEADDRESS = 0x10 + IFA_F_NODAD = 0x2 + IFA_F_OPTIMISTIC = 0x4 + IFA_F_PERMANENT = 0x80 + IFA_F_SECONDARY = 0x1 + IFA_F_TEMPORARY = 0x1 + IFA_F_TENTATIVE = 0x40 + IFA_MAX = 0x7 IFF_ALLMULTI = 0x200 IFF_AUTOMEDIA = 0x4000 IFF_BROADCAST = 0x2 @@ -494,6 +562,53 @@ const ( MS_SYNC = 0x4 MS_SYNCHRONOUS = 0x10 NAME_MAX = 0xff + NETLINK_ADD_MEMBERSHIP = 0x1 + NETLINK_AUDIT = 0x9 + NETLINK_BROADCAST_ERROR = 0x4 + NETLINK_CONNECTOR = 0xb + NETLINK_DNRTMSG = 0xe + NETLINK_DROP_MEMBERSHIP = 0x2 + NETLINK_ECRYPTFS = 0x13 + NETLINK_FIB_LOOKUP = 0xa + NETLINK_FIREWALL = 0x3 + NETLINK_GENERIC = 0x10 + NETLINK_INET_DIAG = 0x4 + NETLINK_IP6_FW = 0xd + NETLINK_ISCSI = 0x8 + NETLINK_KOBJECT_UEVENT = 0xf + NETLINK_NETFILTER = 0xc + NETLINK_NFLOG = 0x5 + NETLINK_NO_ENOBUFS = 0x5 + NETLINK_PKTINFO = 0x3 + NETLINK_ROUTE = 0 + NETLINK_SCSITRANSPORT = 0x12 + NETLINK_SELINUX = 0x7 + NETLINK_UNUSED = 0x1 + NETLINK_USERSOCK = 0x2 + NETLINK_XFRM = 0x6 + NLA_ALIGNTO = 0x4 + NLA_F_NESTED = 0x8000 + NLA_F_NET_BYTEORDER = 0x4000 + NLA_HDRLEN = 0x4 + NLMSG_ALIGNTO = 0x4 + NLMSG_DONE = 0x3 + NLMSG_ERROR = 0x2 + NLMSG_HDRLEN = 0x10 + NLMSG_MIN_TYPE = 0x10 + NLMSG_NOOP = 0x1 + NLMSG_OVERRUN = 0x4 + NLM_F_ACK = 0x4 + NLM_F_APPEND = 0x800 + NLM_F_ATOMIC = 0x400 + NLM_F_CREATE = 0x400 + NLM_F_DUMP = 0x300 + NLM_F_ECHO = 0x8 + NLM_F_EXCL = 0x200 + NLM_F_MATCH = 0x200 + NLM_F_MULTI = 0x2 + NLM_F_REPLACE = 0x100 + NLM_F_REQUEST = 0x1 + NLM_F_ROOT = 0x100 O_ACCMODE = 0x3 O_APPEND = 0x400 O_ASYNC = 0x2000 @@ -590,6 +705,94 @@ const ( PTRACE_SYSEMU = 0x1f PTRACE_SYSEMU_SINGLESTEP = 0x20 PTRACE_TRACEME = 0 + RTAX_ADVMSS = 0x8 + RTAX_CWND = 0x7 + RTAX_FEATURES = 0xc + RTAX_FEATURE_ALLFRAG = 0x8 + RTAX_FEATURE_ECN = 0x1 + RTAX_FEATURE_SACK = 0x2 + RTAX_FEATURE_TIMESTAMP = 0x4 + RTAX_HOPLIMIT = 0xa + RTAX_INITCWND = 0xb + RTAX_LOCK = 0x1 + RTAX_MAX = 0xd + RTAX_MTU = 0x2 + RTAX_REORDERING = 0x9 + RTAX_RTO_MIN = 0xd + RTAX_RTT = 0x4 + RTAX_RTTVAR = 0x5 + RTAX_SSTHRESH = 0x6 + RTAX_UNSPEC = 0 + RTAX_WINDOW = 0x3 + RTA_ALIGNTO = 0x4 + RTA_MAX = 0xf + RTM_BASE = 0x10 + RTM_DELACTION = 0x31 + RTM_DELADDR = 0x15 + RTM_DELADDRLABEL = 0x49 + RTM_DELLINK = 0x11 + RTM_DELNEIGH = 0x1d + RTM_DELQDISC = 0x25 + RTM_DELROUTE = 0x19 + RTM_DELRULE = 0x21 + RTM_DELTCLASS = 0x29 + RTM_DELTFILTER = 0x2d + RTM_F_CLONED = 0x200 + RTM_F_EQUALIZE = 0x400 + RTM_F_NOTIFY = 0x100 + RTM_F_PREFIX = 0x800 + RTM_GETACTION = 0x32 + RTM_GETADDR = 0x16 + RTM_GETADDRLABEL = 0x4a + RTM_GETANYCAST = 0x3e + RTM_GETDCB = 0x4e + RTM_GETLINK = 0x12 + RTM_GETMULTICAST = 0x3a + RTM_GETNEIGH = 0x1e + RTM_GETNEIGHTBL = 0x42 + RTM_GETQDISC = 0x26 + RTM_GETROUTE = 0x1a + RTM_GETRULE = 0x22 + RTM_GETTCLASS = 0x2a + RTM_GETTFILTER = 0x2e + RTM_MAX = 0x4f + RTM_NEWACTION = 0x30 + RTM_NEWADDR = 0x14 + RTM_NEWADDRLABEL = 0x48 + RTM_NEWLINK = 0x10 + RTM_NEWNDUSEROPT = 0x44 + RTM_NEWNEIGH = 0x1c + RTM_NEWNEIGHTBL = 0x40 + RTM_NEWPREFIX = 0x34 + RTM_NEWQDISC = 0x24 + RTM_NEWROUTE = 0x18 + RTM_NEWRULE = 0x20 + RTM_NEWTCLASS = 0x28 + RTM_NEWTFILTER = 0x2c + RTM_NR_FAMILIES = 0x10 + RTM_NR_MSGTYPES = 0x40 + RTM_SETDCB = 0x4f + RTM_SETLINK = 0x13 + RTM_SETNEIGHTBL = 0x43 + RTNH_ALIGNTO = 0x4 + RTNH_F_DEAD = 0x1 + RTNH_F_ONLINK = 0x4 + RTNH_F_PERVASIVE = 0x2 + RTN_MAX = 0xb + RTPROT_BIRD = 0xc + RTPROT_BOOT = 0x3 + RTPROT_DHCP = 0x10 + RTPROT_DNROUTED = 0xd + RTPROT_GATED = 0x8 + RTPROT_KERNEL = 0x2 + RTPROT_MRT = 0xa + RTPROT_NTK = 0xf + RTPROT_RA = 0x9 + RTPROT_REDIRECT = 0x1 + RTPROT_STATIC = 0x4 + RTPROT_UNSPEC = 0 + RTPROT_XORP = 0xe + RTPROT_ZEBRA = 0xb SCM_CREDENTIALS = 0x2 SCM_RIGHTS = 0x1 SCM_TIMESTAMP = 0x1d diff --git a/src/pkg/syscall/zerrors_linux_amd64.go b/src/pkg/syscall/zerrors_linux_amd64.go index d5efdf55d..89260740d 100644 --- a/src/pkg/syscall/zerrors_linux_amd64.go +++ b/src/pkg/syscall/zerrors_linux_amd64.go @@ -48,6 +48,64 @@ const ( AF_UNSPEC = 0 AF_WANPIPE = 0x19 AF_X25 = 0x9 + ARPHRD_ADAPT = 0x108 + ARPHRD_APPLETLK = 0x8 + ARPHRD_ARCNET = 0x7 + ARPHRD_ASH = 0x30d + ARPHRD_ATM = 0x13 + ARPHRD_AX25 = 0x3 + ARPHRD_BIF = 0x307 + ARPHRD_CHAOS = 0x5 + ARPHRD_CISCO = 0x201 + ARPHRD_CSLIP = 0x101 + ARPHRD_CSLIP6 = 0x103 + ARPHRD_DDCMP = 0x205 + ARPHRD_DLCI = 0xf + ARPHRD_ECONET = 0x30e + ARPHRD_EETHER = 0x2 + ARPHRD_ETHER = 0x1 + ARPHRD_EUI64 = 0x1b + ARPHRD_FCAL = 0x311 + ARPHRD_FCFABRIC = 0x313 + ARPHRD_FCPL = 0x312 + ARPHRD_FCPP = 0x310 + ARPHRD_FDDI = 0x306 + ARPHRD_FRAD = 0x302 + ARPHRD_HDLC = 0x201 + ARPHRD_HIPPI = 0x30c + ARPHRD_HWX25 = 0x110 + ARPHRD_IEEE1394 = 0x18 + ARPHRD_IEEE802 = 0x6 + ARPHRD_IEEE80211 = 0x321 + ARPHRD_IEEE80211_PRISM = 0x322 + ARPHRD_IEEE80211_RADIOTAP = 0x323 + ARPHRD_IEEE802154 = 0x324 + ARPHRD_IEEE802154_PHY = 0x325 + ARPHRD_IEEE802_TR = 0x320 + ARPHRD_INFINIBAND = 0x20 + ARPHRD_IPDDP = 0x309 + ARPHRD_IPGRE = 0x30a + ARPHRD_IRDA = 0x30f + ARPHRD_LAPB = 0x204 + ARPHRD_LOCALTLK = 0x305 + ARPHRD_LOOPBACK = 0x304 + ARPHRD_METRICOM = 0x17 + ARPHRD_NETROM = 0 + ARPHRD_NONE = 0xfffe + ARPHRD_PIMREG = 0x30b + ARPHRD_PPP = 0x200 + ARPHRD_PRONET = 0x4 + ARPHRD_RAWHDLC = 0x206 + ARPHRD_ROSE = 0x10e + ARPHRD_RSRVD = 0x104 + ARPHRD_SIT = 0x308 + ARPHRD_SKIP = 0x303 + ARPHRD_SLIP = 0x100 + ARPHRD_SLIP6 = 0x102 + ARPHRD_TUNNEL = 0x300 + ARPHRD_TUNNEL6 = 0x301 + ARPHRD_VOID = 0xffff + ARPHRD_X25 = 0x10f DT_BLK = 0x6 DT_CHR = 0x2 DT_DIR = 0x4 @@ -241,6 +299,16 @@ const ( F_ULOCK = 0 F_UNLCK = 0x2 F_WRLCK = 0x1 + IFA_F_DADFAILED = 0x8 + IFA_F_DEPRECATED = 0x20 + IFA_F_HOMEADDRESS = 0x10 + IFA_F_NODAD = 0x2 + IFA_F_OPTIMISTIC = 0x4 + IFA_F_PERMANENT = 0x80 + IFA_F_SECONDARY = 0x1 + IFA_F_TEMPORARY = 0x1 + IFA_F_TENTATIVE = 0x40 + IFA_MAX = 0x7 IFF_ALLMULTI = 0x200 IFF_AUTOMEDIA = 0x4000 IFF_BROADCAST = 0x2 @@ -494,6 +562,53 @@ const ( MS_SYNC = 0x4 MS_SYNCHRONOUS = 0x10 NAME_MAX = 0xff + NETLINK_ADD_MEMBERSHIP = 0x1 + NETLINK_AUDIT = 0x9 + NETLINK_BROADCAST_ERROR = 0x4 + NETLINK_CONNECTOR = 0xb + NETLINK_DNRTMSG = 0xe + NETLINK_DROP_MEMBERSHIP = 0x2 + NETLINK_ECRYPTFS = 0x13 + NETLINK_FIB_LOOKUP = 0xa + NETLINK_FIREWALL = 0x3 + NETLINK_GENERIC = 0x10 + NETLINK_INET_DIAG = 0x4 + NETLINK_IP6_FW = 0xd + NETLINK_ISCSI = 0x8 + NETLINK_KOBJECT_UEVENT = 0xf + NETLINK_NETFILTER = 0xc + NETLINK_NFLOG = 0x5 + NETLINK_NO_ENOBUFS = 0x5 + NETLINK_PKTINFO = 0x3 + NETLINK_ROUTE = 0 + NETLINK_SCSITRANSPORT = 0x12 + NETLINK_SELINUX = 0x7 + NETLINK_UNUSED = 0x1 + NETLINK_USERSOCK = 0x2 + NETLINK_XFRM = 0x6 + NLA_ALIGNTO = 0x4 + NLA_F_NESTED = 0x8000 + NLA_F_NET_BYTEORDER = 0x4000 + NLA_HDRLEN = 0x4 + NLMSG_ALIGNTO = 0x4 + NLMSG_DONE = 0x3 + NLMSG_ERROR = 0x2 + NLMSG_HDRLEN = 0x10 + NLMSG_MIN_TYPE = 0x10 + NLMSG_NOOP = 0x1 + NLMSG_OVERRUN = 0x4 + NLM_F_ACK = 0x4 + NLM_F_APPEND = 0x800 + NLM_F_ATOMIC = 0x400 + NLM_F_CREATE = 0x400 + NLM_F_DUMP = 0x300 + NLM_F_ECHO = 0x8 + NLM_F_EXCL = 0x200 + NLM_F_MATCH = 0x200 + NLM_F_MULTI = 0x2 + NLM_F_REPLACE = 0x100 + NLM_F_REQUEST = 0x1 + NLM_F_ROOT = 0x100 O_ACCMODE = 0x3 O_APPEND = 0x400 O_ASYNC = 0x2000 @@ -591,6 +706,94 @@ const ( PTRACE_SYSEMU = 0x1f PTRACE_SYSEMU_SINGLESTEP = 0x20 PTRACE_TRACEME = 0 + RTAX_ADVMSS = 0x8 + RTAX_CWND = 0x7 + RTAX_FEATURES = 0xc + RTAX_FEATURE_ALLFRAG = 0x8 + RTAX_FEATURE_ECN = 0x1 + RTAX_FEATURE_SACK = 0x2 + RTAX_FEATURE_TIMESTAMP = 0x4 + RTAX_HOPLIMIT = 0xa + RTAX_INITCWND = 0xb + RTAX_LOCK = 0x1 + RTAX_MAX = 0xd + RTAX_MTU = 0x2 + RTAX_REORDERING = 0x9 + RTAX_RTO_MIN = 0xd + RTAX_RTT = 0x4 + RTAX_RTTVAR = 0x5 + RTAX_SSTHRESH = 0x6 + RTAX_UNSPEC = 0 + RTAX_WINDOW = 0x3 + RTA_ALIGNTO = 0x4 + RTA_MAX = 0xf + RTM_BASE = 0x10 + RTM_DELACTION = 0x31 + RTM_DELADDR = 0x15 + RTM_DELADDRLABEL = 0x49 + RTM_DELLINK = 0x11 + RTM_DELNEIGH = 0x1d + RTM_DELQDISC = 0x25 + RTM_DELROUTE = 0x19 + RTM_DELRULE = 0x21 + RTM_DELTCLASS = 0x29 + RTM_DELTFILTER = 0x2d + RTM_F_CLONED = 0x200 + RTM_F_EQUALIZE = 0x400 + RTM_F_NOTIFY = 0x100 + RTM_F_PREFIX = 0x800 + RTM_GETACTION = 0x32 + RTM_GETADDR = 0x16 + RTM_GETADDRLABEL = 0x4a + RTM_GETANYCAST = 0x3e + RTM_GETDCB = 0x4e + RTM_GETLINK = 0x12 + RTM_GETMULTICAST = 0x3a + RTM_GETNEIGH = 0x1e + RTM_GETNEIGHTBL = 0x42 + RTM_GETQDISC = 0x26 + RTM_GETROUTE = 0x1a + RTM_GETRULE = 0x22 + RTM_GETTCLASS = 0x2a + RTM_GETTFILTER = 0x2e + RTM_MAX = 0x4f + RTM_NEWACTION = 0x30 + RTM_NEWADDR = 0x14 + RTM_NEWADDRLABEL = 0x48 + RTM_NEWLINK = 0x10 + RTM_NEWNDUSEROPT = 0x44 + RTM_NEWNEIGH = 0x1c + RTM_NEWNEIGHTBL = 0x40 + RTM_NEWPREFIX = 0x34 + RTM_NEWQDISC = 0x24 + RTM_NEWROUTE = 0x18 + RTM_NEWRULE = 0x20 + RTM_NEWTCLASS = 0x28 + RTM_NEWTFILTER = 0x2c + RTM_NR_FAMILIES = 0x10 + RTM_NR_MSGTYPES = 0x40 + RTM_SETDCB = 0x4f + RTM_SETLINK = 0x13 + RTM_SETNEIGHTBL = 0x43 + RTNH_ALIGNTO = 0x4 + RTNH_F_DEAD = 0x1 + RTNH_F_ONLINK = 0x4 + RTNH_F_PERVASIVE = 0x2 + RTN_MAX = 0xb + RTPROT_BIRD = 0xc + RTPROT_BOOT = 0x3 + RTPROT_DHCP = 0x10 + RTPROT_DNROUTED = 0xd + RTPROT_GATED = 0x8 + RTPROT_KERNEL = 0x2 + RTPROT_MRT = 0xa + RTPROT_NTK = 0xf + RTPROT_RA = 0x9 + RTPROT_REDIRECT = 0x1 + RTPROT_STATIC = 0x4 + RTPROT_UNSPEC = 0 + RTPROT_XORP = 0xe + RTPROT_ZEBRA = 0xb SCM_CREDENTIALS = 0x2 SCM_RIGHTS = 0x1 SCM_TIMESTAMP = 0x1d diff --git a/src/pkg/syscall/zerrors_linux_arm.go b/src/pkg/syscall/zerrors_linux_arm.go index 356d51ca5..50cdaf18a 100644 --- a/src/pkg/syscall/zerrors_linux_arm.go +++ b/src/pkg/syscall/zerrors_linux_arm.go @@ -16,9 +16,11 @@ const ( AF_AX25 = 0x3 AF_BLUETOOTH = 0x1f AF_BRIDGE = 0x7 + AF_CAN = 0x1d AF_DECnet = 0xc AF_ECONET = 0x13 AF_FILE = 0x1 + AF_IEEE802154 = 0x24 AF_INET = 0x2 AF_INET6 = 0xa AF_IPX = 0x4 @@ -26,22 +28,93 @@ const ( AF_ISDN = 0x22 AF_IUCV = 0x20 AF_KEY = 0xf + AF_LLC = 0x1a AF_LOCAL = 0x1 - AF_MAX = 0x23 + AF_MAX = 0x25 AF_NETBEUI = 0xd AF_NETLINK = 0x10 AF_NETROM = 0x6 AF_PACKET = 0x11 + AF_PHONET = 0x23 AF_PPPOX = 0x18 + AF_RDS = 0x15 AF_ROSE = 0xb AF_ROUTE = 0x10 AF_RXRPC = 0x21 AF_SECURITY = 0xe AF_SNA = 0x16 + AF_TIPC = 0x1e AF_UNIX = 0x1 AF_UNSPEC = 0 AF_WANPIPE = 0x19 AF_X25 = 0x9 + ARPHRD_ADAPT = 0x108 + ARPHRD_APPLETLK = 0x8 + ARPHRD_ARCNET = 0x7 + ARPHRD_ASH = 0x30d + ARPHRD_ATM = 0x13 + ARPHRD_AX25 = 0x3 + ARPHRD_BIF = 0x307 + ARPHRD_CHAOS = 0x5 + ARPHRD_CISCO = 0x201 + ARPHRD_CSLIP = 0x101 + ARPHRD_CSLIP6 = 0x103 + ARPHRD_DDCMP = 0x205 + ARPHRD_DLCI = 0xf + ARPHRD_ECONET = 0x30e + ARPHRD_EETHER = 0x2 + ARPHRD_ETHER = 0x1 + ARPHRD_EUI64 = 0x1b + ARPHRD_FCAL = 0x311 + ARPHRD_FCFABRIC = 0x313 + ARPHRD_FCPL = 0x312 + ARPHRD_FCPP = 0x310 + ARPHRD_FDDI = 0x306 + ARPHRD_FRAD = 0x302 + ARPHRD_HDLC = 0x201 + ARPHRD_HIPPI = 0x30c + ARPHRD_HWX25 = 0x110 + ARPHRD_IEEE1394 = 0x18 + ARPHRD_IEEE802 = 0x6 + ARPHRD_IEEE80211 = 0x321 + ARPHRD_IEEE80211_PRISM = 0x322 + ARPHRD_IEEE80211_RADIOTAP = 0x323 + ARPHRD_IEEE802154 = 0x324 + ARPHRD_IEEE802154_PHY = 0x325 + ARPHRD_IEEE802_TR = 0x320 + ARPHRD_INFINIBAND = 0x20 + ARPHRD_IPDDP = 0x309 + ARPHRD_IPGRE = 0x30a + ARPHRD_IRDA = 0x30f + ARPHRD_LAPB = 0x204 + ARPHRD_LOCALTLK = 0x305 + ARPHRD_LOOPBACK = 0x304 + ARPHRD_METRICOM = 0x17 + ARPHRD_NETROM = 0 + ARPHRD_NONE = 0xfffe + ARPHRD_PIMREG = 0x30b + ARPHRD_PPP = 0x200 + ARPHRD_PRONET = 0x4 + ARPHRD_RAWHDLC = 0x206 + ARPHRD_ROSE = 0x10e + ARPHRD_RSRVD = 0x104 + ARPHRD_SIT = 0x308 + ARPHRD_SKIP = 0x303 + ARPHRD_SLIP = 0x100 + ARPHRD_SLIP6 = 0x102 + ARPHRD_TUNNEL = 0x300 + ARPHRD_TUNNEL6 = 0x301 + ARPHRD_VOID = 0xffff + ARPHRD_X25 = 0x10f + DT_BLK = 0x6 + DT_CHR = 0x2 + DT_DIR = 0x4 + DT_FIFO = 0x1 + DT_LNK = 0xa + DT_REG = 0x8 + DT_SOCK = 0xc + DT_UNKNOWN = 0 + DT_WHT = 0xe E2BIG = 0x7 EACCES = 0xd EADDRINUSE = 0x62 @@ -206,6 +279,7 @@ const ( F_GETLK = 0xc F_GETLK64 = 0xc F_GETOWN = 0x9 + F_GETOWN_EX = 0x10 F_GETSIG = 0xb F_LOCK = 0x1 F_NOTIFY = 0x402 @@ -219,6 +293,7 @@ const ( F_SETLKW = 0xe F_SETLKW64 = 0xe F_SETOWN = 0x8 + F_SETOWN_EX = 0xf F_SETSIG = 0xa F_SHLCK = 0x8 F_TEST = 0x3 @@ -226,6 +301,39 @@ const ( F_ULOCK = 0 F_UNLCK = 0x2 F_WRLCK = 0x1 + IFA_F_DADFAILED = 0x8 + IFA_F_DEPRECATED = 0x20 + IFA_F_HOMEADDRESS = 0x10 + IFA_F_NODAD = 0x2 + IFA_F_OPTIMISTIC = 0x4 + IFA_F_PERMANENT = 0x80 + IFA_F_SECONDARY = 0x1 + IFA_F_TEMPORARY = 0x1 + IFA_F_TENTATIVE = 0x40 + IFA_MAX = 0x7 + IFF_ALLMULTI = 0x200 + IFF_AUTOMEDIA = 0x4000 + IFF_BROADCAST = 0x2 + IFF_DEBUG = 0x4 + IFF_DYNAMIC = 0x8000 + IFF_LOOPBACK = 0x8 + IFF_MASTER = 0x400 + IFF_MULTICAST = 0x1000 + IFF_NOARP = 0x80 + IFF_NOTRAILERS = 0x20 + IFF_NO_PI = 0x1000 + IFF_ONE_QUEUE = 0x2000 + IFF_POINTOPOINT = 0x10 + IFF_PORTSEL = 0x2000 + IFF_PROMISC = 0x100 + IFF_RUNNING = 0x40 + IFF_SLAVE = 0x800 + IFF_TAP = 0x2 + IFF_TUN = 0x1 + IFF_TUN_EXCL = 0x8000 + IFF_UP = 0x1 + IFF_VNET_HDR = 0x4000 + IFNAMSIZ = 0x10 IN_ACCESS = 0x1 IN_ALL_EVENTS = 0xfff IN_ATTRIB = 0x4 @@ -389,9 +497,54 @@ const ( LINUX_REBOOT_CMD_SW_SUSPEND = 0xd000fce2 LINUX_REBOOT_MAGIC1 = 0xfee1dead LINUX_REBOOT_MAGIC2 = 0x28121969 + MADV_DOFORK = 0xb + MADV_DONTFORK = 0xa + MADV_DONTNEED = 0x4 + MADV_HWPOISON = 0x64 + MADV_MERGEABLE = 0xc + MADV_NORMAL = 0 + MADV_RANDOM = 0x1 + MADV_REMOVE = 0x9 + MADV_SEQUENTIAL = 0x2 + MADV_UNMERGEABLE = 0xd + MADV_WILLNEED = 0x3 + MAP_ANON = 0x20 + MAP_ANONYMOUS = 0x20 + MAP_DENYWRITE = 0x800 + MAP_EXECUTABLE = 0x1000 + MAP_FILE = 0 + MAP_FIXED = 0x10 + MAP_GROWSDOWN = 0x100 + MAP_LOCKED = 0x2000 + MAP_NONBLOCK = 0x10000 + MAP_NORESERVE = 0x4000 + MAP_POPULATE = 0x8000 + MAP_PRIVATE = 0x2 + MAP_SHARED = 0x1 + MAP_TYPE = 0xf + MCL_CURRENT = 0x1 + MCL_FUTURE = 0x2 MNT_DETACH = 0x2 MNT_EXPIRE = 0x4 MNT_FORCE = 0x1 + MSG_CMSG_CLOEXEC = 0x40000000 + MSG_CONFIRM = 0x800 + MSG_CTRUNC = 0x8 + MSG_DONTROUTE = 0x4 + MSG_DONTWAIT = 0x40 + MSG_EOR = 0x80 + MSG_ERRQUEUE = 0x2000 + MSG_FIN = 0x200 + MSG_MORE = 0x8000 + MSG_NOSIGNAL = 0x4000 + MSG_OOB = 0x1 + MSG_PEEK = 0x2 + MSG_PROXY = 0x10 + MSG_RST = 0x1000 + MSG_SYN = 0x400 + MSG_TRUNC = 0x20 + MSG_TRYHARD = 0x4 + MSG_WAITALL = 0x100 MS_ASYNC = 0x1 MS_BIND = 0x1000 MS_INVALIDATE = 0x2 @@ -409,6 +562,53 @@ const ( MS_SYNC = 0x4 MS_SYNCHRONOUS = 0x10 NAME_MAX = 0xff + NETLINK_ADD_MEMBERSHIP = 0x1 + NETLINK_AUDIT = 0x9 + NETLINK_BROADCAST_ERROR = 0x4 + NETLINK_CONNECTOR = 0xb + NETLINK_DNRTMSG = 0xe + NETLINK_DROP_MEMBERSHIP = 0x2 + NETLINK_ECRYPTFS = 0x13 + NETLINK_FIB_LOOKUP = 0xa + NETLINK_FIREWALL = 0x3 + NETLINK_GENERIC = 0x10 + NETLINK_INET_DIAG = 0x4 + NETLINK_IP6_FW = 0xd + NETLINK_ISCSI = 0x8 + NETLINK_KOBJECT_UEVENT = 0xf + NETLINK_NETFILTER = 0xc + NETLINK_NFLOG = 0x5 + NETLINK_NO_ENOBUFS = 0x5 + NETLINK_PKTINFO = 0x3 + NETLINK_ROUTE = 0 + NETLINK_SCSITRANSPORT = 0x12 + NETLINK_SELINUX = 0x7 + NETLINK_UNUSED = 0x1 + NETLINK_USERSOCK = 0x2 + NETLINK_XFRM = 0x6 + NLA_ALIGNTO = 0x4 + NLA_F_NESTED = 0x8000 + NLA_F_NET_BYTEORDER = 0x4000 + NLA_HDRLEN = 0x4 + NLMSG_ALIGNTO = 0x4 + NLMSG_DONE = 0x3 + NLMSG_ERROR = 0x2 + NLMSG_HDRLEN = 0x10 + NLMSG_MIN_TYPE = 0x10 + NLMSG_NOOP = 0x1 + NLMSG_OVERRUN = 0x4 + NLM_F_ACK = 0x4 + NLM_F_APPEND = 0x800 + NLM_F_ATOMIC = 0x400 + NLM_F_CREATE = 0x400 + NLM_F_DUMP = 0x300 + NLM_F_ECHO = 0x8 + NLM_F_EXCL = 0x200 + NLM_F_MATCH = 0x200 + NLM_F_MULTI = 0x2 + NLM_F_REPLACE = 0x100 + NLM_F_REQUEST = 0x1 + NLM_F_ROOT = 0x100 O_ACCMODE = 0x3 O_APPEND = 0x400 O_ASYNC = 0x2000 @@ -431,6 +631,27 @@ const ( O_SYNC = 0x1000 O_TRUNC = 0x200 O_WRONLY = 0x1 + PACKET_ADD_MEMBERSHIP = 0x1 + PACKET_BROADCAST = 0x1 + PACKET_DROP_MEMBERSHIP = 0x2 + PACKET_FASTROUTE = 0x6 + PACKET_HOST = 0 + PACKET_LOOPBACK = 0x5 + PACKET_MR_ALLMULTI = 0x2 + PACKET_MR_MULTICAST = 0 + PACKET_MR_PROMISC = 0x1 + PACKET_MULTICAST = 0x2 + PACKET_OTHERHOST = 0x3 + PACKET_OUTGOING = 0x4 + PACKET_RECV_OUTPUT = 0x3 + PACKET_RX_RING = 0x5 + PACKET_STATISTICS = 0x6 + PROT_EXEC = 0x4 + PROT_GROWSDOWN = 0x1000000 + PROT_GROWSUP = 0x2000000 + PROT_NONE = 0 + PROT_READ = 0x1 + PROT_WRITE = 0x2 PTRACE_ATTACH = 0x10 PTRACE_CONT = 0x7 PTRACE_DETACH = 0x11 @@ -475,6 +696,99 @@ const ( PTRACE_SINGLESTEP = 0x9 PTRACE_SYSCALL = 0x18 PTRACE_TRACEME = 0 + RTAX_ADVMSS = 0x8 + RTAX_CWND = 0x7 + RTAX_FEATURES = 0xc + RTAX_FEATURE_ALLFRAG = 0x8 + RTAX_FEATURE_ECN = 0x1 + RTAX_FEATURE_SACK = 0x2 + RTAX_FEATURE_TIMESTAMP = 0x4 + RTAX_HOPLIMIT = 0xa + RTAX_INITCWND = 0xb + RTAX_LOCK = 0x1 + RTAX_MAX = 0xd + RTAX_MTU = 0x2 + RTAX_REORDERING = 0x9 + RTAX_RTO_MIN = 0xd + RTAX_RTT = 0x4 + RTAX_RTTVAR = 0x5 + RTAX_SSTHRESH = 0x6 + RTAX_UNSPEC = 0 + RTAX_WINDOW = 0x3 + RTA_ALIGNTO = 0x4 + RTA_MAX = 0xf + RTM_BASE = 0x10 + RTM_DELACTION = 0x31 + RTM_DELADDR = 0x15 + RTM_DELADDRLABEL = 0x49 + RTM_DELLINK = 0x11 + RTM_DELNEIGH = 0x1d + RTM_DELQDISC = 0x25 + RTM_DELROUTE = 0x19 + RTM_DELRULE = 0x21 + RTM_DELTCLASS = 0x29 + RTM_DELTFILTER = 0x2d + RTM_F_CLONED = 0x200 + RTM_F_EQUALIZE = 0x400 + RTM_F_NOTIFY = 0x100 + RTM_F_PREFIX = 0x800 + RTM_GETACTION = 0x32 + RTM_GETADDR = 0x16 + RTM_GETADDRLABEL = 0x4a + RTM_GETANYCAST = 0x3e + RTM_GETDCB = 0x4e + RTM_GETLINK = 0x12 + RTM_GETMULTICAST = 0x3a + RTM_GETNEIGH = 0x1e + RTM_GETNEIGHTBL = 0x42 + RTM_GETQDISC = 0x26 + RTM_GETROUTE = 0x1a + RTM_GETRULE = 0x22 + RTM_GETTCLASS = 0x2a + RTM_GETTFILTER = 0x2e + RTM_MAX = 0x4f + RTM_NEWACTION = 0x30 + RTM_NEWADDR = 0x14 + RTM_NEWADDRLABEL = 0x48 + RTM_NEWLINK = 0x10 + RTM_NEWNDUSEROPT = 0x44 + RTM_NEWNEIGH = 0x1c + RTM_NEWNEIGHTBL = 0x40 + RTM_NEWPREFIX = 0x34 + RTM_NEWQDISC = 0x24 + RTM_NEWROUTE = 0x18 + RTM_NEWRULE = 0x20 + RTM_NEWTCLASS = 0x28 + RTM_NEWTFILTER = 0x2c + RTM_NR_FAMILIES = 0x10 + RTM_NR_MSGTYPES = 0x40 + RTM_SETDCB = 0x4f + RTM_SETLINK = 0x13 + RTM_SETNEIGHTBL = 0x43 + RTNH_ALIGNTO = 0x4 + RTNH_F_DEAD = 0x1 + RTNH_F_ONLINK = 0x4 + RTNH_F_PERVASIVE = 0x2 + RTN_MAX = 0xb + RTPROT_BIRD = 0xc + RTPROT_BOOT = 0x3 + RTPROT_DHCP = 0x10 + RTPROT_DNROUTED = 0xd + RTPROT_GATED = 0x8 + RTPROT_KERNEL = 0x2 + RTPROT_MRT = 0xa + RTPROT_NTK = 0xf + RTPROT_RA = 0x9 + RTPROT_REDIRECT = 0x1 + RTPROT_STATIC = 0x4 + RTPROT_UNSPEC = 0 + RTPROT_XORP = 0xe + RTPROT_ZEBRA = 0xb + SCM_CREDENTIALS = 0x2 + SCM_RIGHTS = 0x1 + SCM_TIMESTAMP = 0x1d + SCM_TIMESTAMPING = 0x25 + SCM_TIMESTAMPNS = 0x23 SHUT_RD = 0 SHUT_RDWR = 0x2 SHUT_WR = 0x1 @@ -513,6 +827,64 @@ const ( SIGWINCH = 0x1c SIGXCPU = 0x18 SIGXFSZ = 0x19 + SIOCADDDLCI = 0x8980 + SIOCADDMULTI = 0x8931 + SIOCADDRT = 0x890b + SIOCATMARK = 0x8905 + SIOCDARP = 0x8953 + SIOCDELDLCI = 0x8981 + SIOCDELMULTI = 0x8932 + SIOCDELRT = 0x890c + SIOCDEVPRIVATE = 0x89f0 + SIOCDIFADDR = 0x8936 + SIOCDRARP = 0x8960 + SIOCGARP = 0x8954 + SIOCGIFADDR = 0x8915 + SIOCGIFBR = 0x8940 + SIOCGIFBRDADDR = 0x8919 + SIOCGIFCONF = 0x8912 + SIOCGIFCOUNT = 0x8938 + SIOCGIFDSTADDR = 0x8917 + SIOCGIFENCAP = 0x8925 + SIOCGIFFLAGS = 0x8913 + SIOCGIFHWADDR = 0x8927 + SIOCGIFINDEX = 0x8933 + SIOCGIFMAP = 0x8970 + SIOCGIFMEM = 0x891f + SIOCGIFMETRIC = 0x891d + SIOCGIFMTU = 0x8921 + SIOCGIFNAME = 0x8910 + SIOCGIFNETMASK = 0x891b + SIOCGIFPFLAGS = 0x8935 + SIOCGIFSLAVE = 0x8929 + SIOCGIFTXQLEN = 0x8942 + SIOCGPGRP = 0x8904 + SIOCGRARP = 0x8961 + SIOCGSTAMP = 0x8906 + SIOCGSTAMPNS = 0x8907 + SIOCPROTOPRIVATE = 0x89e0 + SIOCRTMSG = 0x890d + SIOCSARP = 0x8955 + SIOCSIFADDR = 0x8916 + SIOCSIFBR = 0x8941 + SIOCSIFBRDADDR = 0x891a + SIOCSIFDSTADDR = 0x8918 + SIOCSIFENCAP = 0x8926 + SIOCSIFFLAGS = 0x8914 + SIOCSIFHWADDR = 0x8924 + SIOCSIFHWBROADCAST = 0x8937 + SIOCSIFLINK = 0x8911 + SIOCSIFMAP = 0x8971 + SIOCSIFMEM = 0x8920 + SIOCSIFMETRIC = 0x891e + SIOCSIFMTU = 0x8922 + SIOCSIFNAME = 0x8923 + SIOCSIFNETMASK = 0x891c + SIOCSIFPFLAGS = 0x8934 + SIOCSIFSLAVE = 0x8930 + SIOCSIFTXQLEN = 0x8943 + SIOCSPGRP = 0x8902 + SIOCSRARP = 0x8962 SOCK_CLOEXEC = 0x80000 SOCK_DCCP = 0x6 SOCK_DGRAM = 0x2 @@ -542,6 +914,7 @@ const ( SO_BSDCOMPAT = 0xe SO_DEBUG = 0x1 SO_DETACH_FILTER = 0x1b + SO_DOMAIN = 0x27 SO_DONTROUTE = 0x5 SO_ERROR = 0x4 SO_KEEPALIVE = 0x9 @@ -555,6 +928,7 @@ const ( SO_PEERNAME = 0x1c SO_PEERSEC = 0x1f SO_PRIORITY = 0xc + SO_PROTOCOL = 0x26 SO_RCVBUF = 0x8 SO_RCVBUFFORCE = 0x21 SO_RCVLOWAT = 0x12 @@ -619,6 +993,19 @@ const ( TCP_QUICKACK = 0xc TCP_SYNCNT = 0x7 TCP_WINDOW_CLAMP = 0xa + TUNGETFEATURES = 0x800454cf + TUNGETIFF = 0x800454d2 + TUNGETSNDBUF = 0x800454d3 + TUNSETDEBUG = 0x400454c9 + TUNSETGROUP = 0x400454ce + TUNSETIFF = 0x400454ca + TUNSETLINK = 0x400454cd + TUNSETNOCSUM = 0x400454c8 + TUNSETOFFLOAD = 0x400454d0 + TUNSETOWNER = 0x400454cc + TUNSETPERSIST = 0x400454cb + TUNSETSNDBUF = 0x400454d4 + TUNSETTXFILTER = 0x400454d1 WALL = 0x40000000 WCLONE = 0x80000000 WCONTINUED = 0x8 @@ -636,134 +1023,134 @@ const ( // Error table var errors = [...]string{ + 1: "operation not permitted", + 2: "no such file or directory", + 3: "no such process", + 4: "interrupted system call", + 5: "input/output error", + 6: "no such device or address", 7: "argument list too long", - 13: "permission denied", - 98: "address already in use", - 99: "cannot assign requested address", - 68: "advertise error", - 97: "address family not supported by protocol", - 11: "resource temporarily unavailable", - 114: "operation already in progress", - 52: "invalid exchange", + 8: "exec format error", 9: "bad file descriptor", - 77: "file descriptor in bad state", - 74: "bad message", - 53: "invalid request descriptor", - 56: "invalid request code", - 57: "invalid slot", - 59: "bad font file format", - 16: "device or resource busy", - 125: "operation canceled", 10: "no child processes", - 44: "channel number out of range", - 70: "communication error on send", - 103: "software caused connection abort", - 111: "connection refused", - 104: "connection reset by peer", - 35: "resource deadlock avoided", - 89: "destination address required", - 33: "numerical argument out of domain", - 73: "RFS specific error", - 122: "disk quota exceeded", - 17: "file exists", + 11: "resource temporarily unavailable", + 12: "cannot allocate memory", + 13: "permission denied", 14: "bad address", + 15: "block device required", + 16: "device or resource busy", + 17: "file exists", + 18: "invalid cross-device link", + 19: "no such device", + 20: "not a directory", + 21: "is a directory", + 22: "invalid argument", + 23: "too many open files in system", + 24: "too many open files", + 25: "inappropriate ioctl for device", + 26: "text file busy", 27: "file too large", - 112: "host is down", - 113: "no route to host", + 28: "no space left on device", + 29: "illegal seek", + 30: "read-only file system", + 31: "too many links", + 32: "broken pipe", + 33: "numerical argument out of domain", + 34: "numerical result out of range", + 35: "resource deadlock avoided", + 36: "file name too long", + 37: "no locks available", + 38: "function not implemented", + 39: "directory not empty", + 40: "too many levels of symbolic links", + 42: "no message of desired type", 43: "identifier removed", - 84: "invalid or incomplete multibyte or wide character", - 115: "operation now in progress", - 4: "interrupted system call", - 22: "invalid argument", - 5: "input/output error", - 106: "transport endpoint is already connected", - 21: "is a directory", - 120: "is a named type file", - 127: "key has expired", - 129: "key was rejected by service", - 128: "key has been revoked", - 51: "level 2 halted", + 44: "channel number out of range", 45: "level 2 not synchronized", 46: "level 3 halted", 47: "level 3 reset", - 79: "can not access a needed shared library", - 80: "accessing a corrupted shared library", - 83: "cannot exec a shared library directly", - 82: "attempting to link in too many shared libraries", - 81: ".lib section in a.out corrupted", 48: "link number out of range", - 40: "too many levels of symbolic links", - 124: "wrong medium type", - 24: "too many open files", - 31: "too many links", - 90: "message too long", - 72: "multihop attempted", - 36: "file name too long", - 119: "no XENIX semaphores available", - 100: "network is down", - 102: "network dropped connection on reset", - 101: "network is unreachable", - 23: "too many open files in system", - 55: "no anode", - 105: "no buffer space available", + 49: "protocol driver not attached", 50: "no CSI structure available", + 51: "level 2 halted", + 52: "invalid exchange", + 53: "invalid request descriptor", + 54: "exchange full", + 55: "no anode", + 56: "invalid request code", + 57: "invalid slot", + 59: "bad font file format", + 60: "device not a stream", 61: "no data available", - 19: "no such device", - 2: "no such file or directory", - 8: "exec format error", - 126: "required key not available", - 37: "no locks available", - 67: "link has been severed", - 123: "no medium found", - 12: "cannot allocate memory", - 42: "no message of desired type", + 62: "timer expired", + 63: "out of streams resources", 64: "machine is not on the network", 65: "package not installed", - 92: "protocol not available", - 28: "no space left on device", - 63: "out of streams resources", - 60: "device not a stream", - 38: "function not implemented", - 15: "block device required", - 107: "transport endpoint is not connected", - 20: "not a directory", - 39: "directory not empty", - 118: "not a XENIX named type file", - 131: "state not recoverable", - 88: "socket operation on non-socket", - 95: "operation not supported", - 25: "inappropriate ioctl for device", - 76: "name not unique on network", - 6: "no such device or address", - 75: "value too large for defined data type", - 130: "owner died", - 1: "operation not permitted", - 96: "protocol family not supported", - 32: "broken pipe", + 66: "object is remote", + 67: "link has been severed", + 68: "advertise error", + 69: "srmount error", + 70: "communication error on send", 71: "protocol error", - 93: "protocol not supported", - 91: "protocol wrong type for socket", - 34: "numerical result out of range", + 72: "multihop attempted", + 73: "RFS specific error", + 74: "bad message", + 75: "value too large for defined data type", + 76: "name not unique on network", + 77: "file descriptor in bad state", 78: "remote address changed", - 66: "object is remote", - 121: "remote I/O error", + 79: "can not access a needed shared library", + 80: "accessing a corrupted shared library", + 81: ".lib section in a.out corrupted", + 82: "attempting to link in too many shared libraries", + 83: "cannot exec a shared library directly", + 84: "invalid or incomplete multibyte or wide character", 85: "interrupted system call should be restarted", - 132: "unknown error 132", - 30: "read-only file system", - 108: "cannot send after transport endpoint shutdown", - 94: "socket type not supported", - 29: "illegal seek", - 3: "no such process", - 69: "srmount error", - 116: "stale NFS file handle", 86: "streams pipe error", - 62: "timer expired", - 110: "connection timed out", + 87: "too many users", + 88: "socket operation on non-socket", + 89: "destination address required", + 90: "message too long", + 91: "protocol wrong type for socket", + 92: "protocol not available", + 93: "protocol not supported", + 94: "socket type not supported", + 95: "operation not supported", + 96: "protocol family not supported", + 97: "address family not supported by protocol", + 98: "address already in use", + 99: "cannot assign requested address", + 100: "network is down", + 101: "network is unreachable", + 102: "network dropped connection on reset", + 103: "software caused connection abort", + 104: "connection reset by peer", + 105: "no buffer space available", + 106: "transport endpoint is already connected", + 107: "transport endpoint is not connected", + 108: "cannot send after transport endpoint shutdown", 109: "too many references: cannot splice", - 26: "text file busy", + 110: "connection timed out", + 111: "connection refused", + 112: "host is down", + 113: "no route to host", + 114: "operation already in progress", + 115: "operation now in progress", + 116: "stale NFS file handle", 117: "structure needs cleaning", - 49: "protocol driver not attached", - 87: "too many users", - 18: "invalid cross-device link", - 54: "exchange full", + 118: "not a XENIX named type file", + 119: "no XENIX semaphores available", + 120: "is a named type file", + 121: "remote I/O error", + 122: "disk quota exceeded", + 123: "no medium found", + 124: "wrong medium type", + 125: "operation canceled", + 126: "required key not available", + 127: "key has expired", + 128: "key has been revoked", + 129: "key was rejected by service", + 130: "owner died", + 131: "state not recoverable", + 132: "unknown error 132", } diff --git a/src/pkg/syscall/zsyscall_linux_386.go b/src/pkg/syscall/zsyscall_linux_386.go index 4f331aa22..8df29f136 100644 --- a/src/pkg/syscall/zsyscall_linux_386.go +++ b/src/pkg/syscall/zsyscall_linux_386.go @@ -971,6 +971,15 @@ func Pwrite(fd int, p []byte, offset int64) (n int, errno int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func Sendfile(outfd int, infd int, offset *int64, count int) (written int, errno int) { + r0, _, e1 := Syscall6(SYS_SENDFILE64, uintptr(outfd), uintptr(infd), uintptr(unsafe.Pointer(offset)), uintptr(count), 0, 0) + written = int(r0) + errno = int(e1) + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Setfsgid(gid int) (errno int) { _, _, e1 := Syscall(SYS_SETFSGID32, uintptr(gid), 0, 0) errno = int(e1) diff --git a/src/pkg/syscall/zsyscall_linux_amd64.go b/src/pkg/syscall/zsyscall_linux_amd64.go index 19501dbfa..d6e287967 100644 --- a/src/pkg/syscall/zsyscall_linux_amd64.go +++ b/src/pkg/syscall/zsyscall_linux_amd64.go @@ -1005,6 +1005,15 @@ func Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func Sendfile(outfd int, infd int, offset *int64, count int) (written int, errno int) { + r0, _, e1 := Syscall6(SYS_SENDFILE, uintptr(outfd), uintptr(infd), uintptr(unsafe.Pointer(offset)), uintptr(count), 0, 0) + written = int(r0) + errno = int(e1) + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Setfsgid(gid int) (errno int) { _, _, e1 := Syscall(SYS_SETFSGID, uintptr(gid), 0, 0) errno = int(e1) diff --git a/src/pkg/syscall/zsyscall_linux_arm.go b/src/pkg/syscall/zsyscall_linux_arm.go index db49b6482..af5f7c50c 100644 --- a/src/pkg/syscall/zsyscall_linux_arm.go +++ b/src/pkg/syscall/zsyscall_linux_arm.go @@ -1070,6 +1070,15 @@ func Lstat(path string, stat *Stat_t) (errno int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func Sendfile(outfd int, infd int, offset *int64, count int) (written int, errno int) { + r0, _, e1 := Syscall6(SYS_SENDFILE64, uintptr(outfd), uintptr(infd), uintptr(unsafe.Pointer(offset)), uintptr(count), 0, 0) + written = int(r0) + errno = int(e1) + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, errno int) { r0, _, e1 := Syscall6(SYS__NEWSELECT, uintptr(nfd), uintptr(unsafe.Pointer(r)), uintptr(unsafe.Pointer(w)), uintptr(unsafe.Pointer(e)), uintptr(unsafe.Pointer(timeout)), 0) n = int(r0) diff --git a/src/pkg/syscall/zsyscall_windows_386.go b/src/pkg/syscall/zsyscall_windows_386.go index f4cfdeed8..447b09043 100644 --- a/src/pkg/syscall/zsyscall_windows_386.go +++ b/src/pkg/syscall/zsyscall_windows_386.go @@ -46,6 +46,7 @@ var ( procCancelIo = getSysProcAddr(modkernel32, "CancelIo") procCreateProcessW = getSysProcAddr(modkernel32, "CreateProcessW") procOpenProcess = getSysProcAddr(modkernel32, "OpenProcess") + procTerminateProcess = getSysProcAddr(modkernel32, "TerminateProcess") procGetExitCodeProcess = getSysProcAddr(modkernel32, "GetExitCodeProcess") procGetStartupInfoW = getSysProcAddr(modkernel32, "GetStartupInfoW") procGetCurrentProcess = getSysProcAddr(modkernel32, "GetCurrentProcess") @@ -70,6 +71,12 @@ var ( procSetHandleInformation = getSysProcAddr(modkernel32, "SetHandleInformation") procFlushFileBuffers = getSysProcAddr(modkernel32, "FlushFileBuffers") procGetFullPathNameW = getSysProcAddr(modkernel32, "GetFullPathNameW") + procCreateFileMappingW = getSysProcAddr(modkernel32, "CreateFileMappingW") + procMapViewOfFile = getSysProcAddr(modkernel32, "MapViewOfFile") + procUnmapViewOfFile = getSysProcAddr(modkernel32, "UnmapViewOfFile") + procFlushViewOfFile = getSysProcAddr(modkernel32, "FlushViewOfFile") + procVirtualLock = getSysProcAddr(modkernel32, "VirtualLock") + procVirtualUnlock = getSysProcAddr(modkernel32, "VirtualUnlock") procWSAStartup = getSysProcAddr(modwsock32, "WSAStartup") procWSACleanup = getSysProcAddr(modwsock32, "WSACleanup") procsocket = getSysProcAddr(modwsock32, "socket") @@ -536,7 +543,7 @@ func CreateProcess(appName *uint16, commandLine *uint16, procSecurity *SecurityA return } -func OpenProcess(da uint32, inheritHandle bool, pid uint32) (handle uint32, errno int) { +func OpenProcess(da uint32, inheritHandle bool, pid uint32) (handle int32, errno int) { var _p0 uint32 if inheritHandle { _p0 = 1 @@ -544,7 +551,7 @@ func OpenProcess(da uint32, inheritHandle bool, pid uint32) (handle uint32, errn _p0 = 0 } r0, _, e1 := Syscall(procOpenProcess, 3, uintptr(da), uintptr(_p0), uintptr(pid)) - handle = uint32(r0) + handle = int32(r0) if handle == 0 { if e1 != 0 { errno = int(e1) @@ -557,7 +564,21 @@ func OpenProcess(da uint32, inheritHandle bool, pid uint32) (handle uint32, errn return } -func GetExitCodeProcess(handle uint32, exitcode *uint32) (errno int) { +func TerminateProcess(handle int32, exitcode uint32) (errno int) { + r1, _, e1 := Syscall(procTerminateProcess, 2, uintptr(handle), uintptr(exitcode), 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = EINVAL + } + } else { + errno = 0 + } + return +} + +func GetExitCodeProcess(handle int32, exitcode *uint32) (errno int) { r1, _, e1 := Syscall(procGetExitCodeProcess, 2, uintptr(handle), uintptr(unsafe.Pointer(exitcode)), 0) if int(r1) == 0 { if e1 != 0 { @@ -901,6 +922,92 @@ func GetFullPathName(path *uint16, buflen uint32, buf *uint16, fname **uint16) ( return } +func CreateFileMapping(fhandle int32, sa *SecurityAttributes, prot uint32, maxSizeHigh uint32, maxSizeLow uint32, name *uint16) (handle int32, errno int) { + r0, _, e1 := Syscall6(procCreateFileMappingW, 6, uintptr(fhandle), uintptr(unsafe.Pointer(sa)), uintptr(prot), uintptr(maxSizeHigh), uintptr(maxSizeLow), uintptr(unsafe.Pointer(name))) + handle = int32(r0) + if handle == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = EINVAL + } + } else { + errno = 0 + } + return +} + +func MapViewOfFile(handle int32, access uint32, offsetHigh uint32, offsetLow uint32, length uintptr) (addr uintptr, errno int) { + r0, _, e1 := Syscall6(procMapViewOfFile, 5, uintptr(handle), uintptr(access), uintptr(offsetHigh), uintptr(offsetLow), uintptr(length), 0) + addr = uintptr(r0) + if addr == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = EINVAL + } + } else { + errno = 0 + } + return +} + +func UnmapViewOfFile(addr uintptr) (errno int) { + r1, _, e1 := Syscall(procUnmapViewOfFile, 1, uintptr(addr), 0, 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = EINVAL + } + } else { + errno = 0 + } + return +} + +func FlushViewOfFile(addr uintptr, length uintptr) (errno int) { + r1, _, e1 := Syscall(procFlushViewOfFile, 2, uintptr(addr), uintptr(length), 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = EINVAL + } + } else { + errno = 0 + } + return +} + +func VirtualLock(addr uintptr, length uintptr) (errno int) { + r1, _, e1 := Syscall(procVirtualLock, 2, uintptr(addr), uintptr(length), 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = EINVAL + } + } else { + errno = 0 + } + return +} + +func VirtualUnlock(addr uintptr, length uintptr) (errno int) { + r1, _, e1 := Syscall(procVirtualUnlock, 2, uintptr(addr), uintptr(length), 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = EINVAL + } + } else { + errno = 0 + } + return +} + func WSAStartup(verreq uint32, data *WSAData) (sockerrno int) { r0, _, _ := Syscall(procWSAStartup, 2, uintptr(verreq), uintptr(unsafe.Pointer(data)), 0) sockerrno = int(r0) diff --git a/src/pkg/syscall/ztypes_darwin_386.go b/src/pkg/syscall/ztypes_darwin_386.go index b3541778e..ba6e590c4 100644 --- a/src/pkg/syscall/ztypes_darwin_386.go +++ b/src/pkg/syscall/ztypes_darwin_386.go @@ -18,9 +18,11 @@ const ( SizeofSockaddrUnix = 0x6a SizeofSockaddrDatalink = 0x14 SizeofLinger = 0x8 - SizeofIpMreq = 0x8 + SizeofIPMreq = 0x8 + SizeofIPv6Mreq = 0x14 SizeofMsghdr = 0x1c SizeofCmsghdr = 0xc + SizeofInet6Pktinfo = 0x14 PTRACE_TRACEME = 0 PTRACE_CONT = 0x7 PTRACE_KILL = 0x8 @@ -222,11 +224,16 @@ type Iovec struct { Len uint32 } -type IpMreq struct { +type IPMreq struct { Multiaddr [4]byte /* in_addr */ Interface [4]byte /* in_addr */ } +type IPv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + type Msghdr struct { Name *byte Namelen uint32 @@ -243,6 +250,11 @@ type Cmsghdr struct { Type int32 } +type Inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + type Kevent_t struct { Ident uint32 Filter int16 diff --git a/src/pkg/syscall/ztypes_darwin_amd64.go b/src/pkg/syscall/ztypes_darwin_amd64.go index d61c8b8de..59c832812 100644 --- a/src/pkg/syscall/ztypes_darwin_amd64.go +++ b/src/pkg/syscall/ztypes_darwin_amd64.go @@ -18,9 +18,11 @@ const ( SizeofSockaddrUnix = 0x6a SizeofSockaddrDatalink = 0x14 SizeofLinger = 0x8 - SizeofIpMreq = 0x8 + SizeofIPMreq = 0x8 + SizeofIPv6Mreq = 0x14 SizeofMsghdr = 0x30 SizeofCmsghdr = 0xc + SizeofInet6Pktinfo = 0x14 PTRACE_TRACEME = 0 PTRACE_CONT = 0x7 PTRACE_KILL = 0x8 @@ -230,11 +232,16 @@ type Iovec struct { Len uint64 } -type IpMreq struct { +type IPMreq struct { Multiaddr [4]byte /* in_addr */ Interface [4]byte /* in_addr */ } +type IPv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + type Msghdr struct { Name *byte Namelen uint32 @@ -253,6 +260,11 @@ type Cmsghdr struct { Type int32 } +type Inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + type Kevent_t struct { Ident uint64 Filter int16 diff --git a/src/pkg/syscall/ztypes_freebsd_386.go b/src/pkg/syscall/ztypes_freebsd_386.go index f4d256f4e..6304d3b81 100644 --- a/src/pkg/syscall/ztypes_freebsd_386.go +++ b/src/pkg/syscall/ztypes_freebsd_386.go @@ -32,9 +32,11 @@ const ( SizeofSockaddrUnix = 0x6a SizeofSockaddrDatalink = 0x36 SizeofLinger = 0x8 - SizeofIpMreq = 0x8 + SizeofIPMreq = 0x8 + SizeofIPv6Mreq = 0x14 SizeofMsghdr = 0x1c SizeofCmsghdr = 0xc + SizeofInet6Pktinfo = 0x14 PTRACE_TRACEME = 0 PTRACE_CONT = 0x7 PTRACE_KILL = 0x8 @@ -219,11 +221,16 @@ type Iovec struct { Len uint32 } -type IpMreq struct { +type IPMreq struct { Multiaddr [4]byte /* in_addr */ Interface [4]byte /* in_addr */ } +type IPv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + type Msghdr struct { Name *byte Namelen uint32 @@ -240,6 +247,11 @@ type Cmsghdr struct { Type int32 } +type Inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + type Kevent_t struct { Ident uint32 Filter int16 diff --git a/src/pkg/syscall/ztypes_freebsd_amd64.go b/src/pkg/syscall/ztypes_freebsd_amd64.go index cf6355caf..ef5a51c4d 100644 --- a/src/pkg/syscall/ztypes_freebsd_amd64.go +++ b/src/pkg/syscall/ztypes_freebsd_amd64.go @@ -32,9 +32,11 @@ const ( SizeofSockaddrUnix = 0x6a SizeofSockaddrDatalink = 0x36 SizeofLinger = 0x8 - SizeofIpMreq = 0x8 + SizeofIPMreq = 0x8 + SizeofIPv6Mreq = 0x14 SizeofMsghdr = 0x30 SizeofCmsghdr = 0xc + SizeofInet6Pktinfo = 0x14 PTRACE_TRACEME = 0 PTRACE_CONT = 0x7 PTRACE_KILL = 0x8 @@ -220,11 +222,16 @@ type Iovec struct { Len uint64 } -type IpMreq struct { +type IPMreq struct { Multiaddr [4]byte /* in_addr */ Interface [4]byte /* in_addr */ } +type IPv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + type Msghdr struct { Name *byte Namelen uint32 @@ -243,6 +250,11 @@ type Cmsghdr struct { Type int32 } +type Inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + type Kevent_t struct { Ident uint64 Filter int16 diff --git a/src/pkg/syscall/ztypes_linux_386.go b/src/pkg/syscall/ztypes_linux_386.go index d98d6af05..65c8b87db 100644 --- a/src/pkg/syscall/ztypes_linux_386.go +++ b/src/pkg/syscall/ztypes_linux_386.go @@ -17,11 +17,89 @@ const ( SizeofSockaddrAny = 0x70 SizeofSockaddrUnix = 0x6e SizeofSockaddrLinklayer = 0x14 + SizeofSockaddrNetlink = 0xc SizeofLinger = 0x8 - SizeofIpMreq = 0x8 + SizeofIPMreq = 0x8 + SizeofIPv6Mreq = 0x14 SizeofMsghdr = 0x1c SizeofCmsghdr = 0xc + SizeofInet6Pktinfo = 0x14 SizeofUcred = 0xc + IFA_UNSPEC = 0 + IFA_ADDRESS = 0x1 + IFA_LOCAL = 0x2 + IFA_LABEL = 0x3 + IFA_BROADCAST = 0x4 + IFA_ANYCAST = 0x5 + IFA_CACHEINFO = 0x6 + IFA_MULTICAST = 0x7 + IFLA_UNSPEC = 0 + IFLA_ADDRESS = 0x1 + IFLA_BROADCAST = 0x2 + IFLA_IFNAME = 0x3 + IFLA_MTU = 0x4 + IFLA_LINK = 0x5 + IFLA_QDISC = 0x6 + IFLA_STATS = 0x7 + IFLA_COST = 0x8 + IFLA_PRIORITY = 0x9 + IFLA_MASTER = 0xa + IFLA_WIRELESS = 0xb + IFLA_PROTINFO = 0xc + IFLA_TXQLEN = 0xd + IFLA_MAP = 0xe + IFLA_WEIGHT = 0xf + IFLA_OPERSTATE = 0x10 + IFLA_LINKMODE = 0x11 + IFLA_LINKINFO = 0x12 + IFLA_NET_NS_PID = 0x13 + IFLA_IFALIAS = 0x14 + IFLA_MAX = 0x14 + RT_SCOPE_UNIVERSE = 0 + RT_SCOPE_SITE = 0xc8 + RT_SCOPE_LINK = 0xfd + RT_SCOPE_HOST = 0xfe + RT_SCOPE_NOWHERE = 0xff + RT_TABLE_UNSPEC = 0 + RT_TABLE_COMPAT = 0xfc + RT_TABLE_DEFAULT = 0xfd + RT_TABLE_MAIN = 0xfe + RT_TABLE_LOCAL = 0xff + RT_TABLE_MAX = 0xffffffff + RTA_UNSPEC = 0 + RTA_DST = 0x1 + RTA_SRC = 0x2 + RTA_IIF = 0x3 + RTA_OIF = 0x4 + RTA_GATEWAY = 0x5 + RTA_PRIORITY = 0x6 + RTA_PREFSRC = 0x7 + RTA_METRICS = 0x8 + RTA_MULTIPATH = 0x9 + RTA_FLOW = 0xb + RTA_CACHEINFO = 0xc + RTA_TABLE = 0xf + RTN_UNSPEC = 0 + RTN_UNICAST = 0x1 + RTN_LOCAL = 0x2 + RTN_BROADCAST = 0x3 + RTN_ANYCAST = 0x4 + RTN_MULTICAST = 0x5 + RTN_BLACKHOLE = 0x6 + RTN_UNREACHABLE = 0x7 + RTN_PROHIBIT = 0x8 + RTN_THROW = 0x9 + RTN_NAT = 0xa + RTN_XRESOLVE = 0xb + SizeofNlMsghdr = 0x10 + SizeofNlMsgerr = 0x14 + SizeofRtGenmsg = 0x1 + SizeofNlAttr = 0x4 + SizeofRtAttr = 0x4 + SizeofIfInfomsg = 0x10 + SizeofIfAddrmsg = 0x8 + SizeofRtmsg = 0xc + SizeofRtNexthop = 0x8 SizeofInotifyEvent = 0x10 ) @@ -193,6 +271,13 @@ type RawSockaddrLinklayer struct { Addr [8]uint8 } +type RawSockaddrNetlink struct { + Family uint16 + Pad uint16 + Pid uint32 + Groups uint32 +} + type RawSockaddr struct { Family uint16 Data [14]int8 @@ -215,11 +300,16 @@ type Iovec struct { Len uint32 } -type IpMreq struct { +type IPMreq struct { Multiaddr [4]byte /* in_addr */ Interface [4]byte /* in_addr */ } +type IPv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + type Msghdr struct { Name *byte Namelen uint32 @@ -236,12 +326,80 @@ type Cmsghdr struct { Type int32 } +type Inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + type Ucred struct { Pid int32 Uid uint32 Gid uint32 } +type NlMsghdr struct { + Len uint32 + Type uint16 + Flags uint16 + Seq uint32 + Pid uint32 +} + +type NlMsgerr struct { + Error int32 + Msg NlMsghdr +} + +type RtGenmsg struct { + Family uint8 +} + +type NlAttr struct { + Len uint16 + Type uint16 +} + +type RtAttr struct { + Len uint16 + Type uint16 +} + +type IfInfomsg struct { + Family uint8 + X__ifi_pad uint8 + Type uint16 + Index int32 + Flags uint32 + Change uint32 +} + +type IfAddrmsg struct { + Family uint8 + Prefixlen uint8 + Flags uint8 + Scope uint8 + Index uint32 +} + +type RtMsg struct { + Family uint8 + Dst_len uint8 + Src_len uint8 + Tos uint8 + Table uint8 + Protocol uint8 + Scope uint8 + Type uint8 + Flags uint32 +} + +type RtNexthop struct { + Len uint16 + Flags uint8 + Hops uint8 + Ifindex int32 +} + type InotifyEvent struct { Wd int32 Mask uint32 diff --git a/src/pkg/syscall/ztypes_linux_amd64.go b/src/pkg/syscall/ztypes_linux_amd64.go index db5c32cae..e26b6bfd2 100644 --- a/src/pkg/syscall/ztypes_linux_amd64.go +++ b/src/pkg/syscall/ztypes_linux_amd64.go @@ -17,11 +17,89 @@ const ( SizeofSockaddrAny = 0x70 SizeofSockaddrUnix = 0x6e SizeofSockaddrLinklayer = 0x14 + SizeofSockaddrNetlink = 0xc SizeofLinger = 0x8 - SizeofIpMreq = 0x8 + SizeofIPMreq = 0x8 + SizeofIPv6Mreq = 0x14 SizeofMsghdr = 0x38 SizeofCmsghdr = 0x10 + SizeofInet6Pktinfo = 0x14 SizeofUcred = 0xc + IFA_UNSPEC = 0 + IFA_ADDRESS = 0x1 + IFA_LOCAL = 0x2 + IFA_LABEL = 0x3 + IFA_BROADCAST = 0x4 + IFA_ANYCAST = 0x5 + IFA_CACHEINFO = 0x6 + IFA_MULTICAST = 0x7 + IFLA_UNSPEC = 0 + IFLA_ADDRESS = 0x1 + IFLA_BROADCAST = 0x2 + IFLA_IFNAME = 0x3 + IFLA_MTU = 0x4 + IFLA_LINK = 0x5 + IFLA_QDISC = 0x6 + IFLA_STATS = 0x7 + IFLA_COST = 0x8 + IFLA_PRIORITY = 0x9 + IFLA_MASTER = 0xa + IFLA_WIRELESS = 0xb + IFLA_PROTINFO = 0xc + IFLA_TXQLEN = 0xd + IFLA_MAP = 0xe + IFLA_WEIGHT = 0xf + IFLA_OPERSTATE = 0x10 + IFLA_LINKMODE = 0x11 + IFLA_LINKINFO = 0x12 + IFLA_NET_NS_PID = 0x13 + IFLA_IFALIAS = 0x14 + IFLA_MAX = 0x14 + RT_SCOPE_UNIVERSE = 0 + RT_SCOPE_SITE = 0xc8 + RT_SCOPE_LINK = 0xfd + RT_SCOPE_HOST = 0xfe + RT_SCOPE_NOWHERE = 0xff + RT_TABLE_UNSPEC = 0 + RT_TABLE_COMPAT = 0xfc + RT_TABLE_DEFAULT = 0xfd + RT_TABLE_MAIN = 0xfe + RT_TABLE_LOCAL = 0xff + RT_TABLE_MAX = 0xffffffff + RTA_UNSPEC = 0 + RTA_DST = 0x1 + RTA_SRC = 0x2 + RTA_IIF = 0x3 + RTA_OIF = 0x4 + RTA_GATEWAY = 0x5 + RTA_PRIORITY = 0x6 + RTA_PREFSRC = 0x7 + RTA_METRICS = 0x8 + RTA_MULTIPATH = 0x9 + RTA_FLOW = 0xb + RTA_CACHEINFO = 0xc + RTA_TABLE = 0xf + RTN_UNSPEC = 0 + RTN_UNICAST = 0x1 + RTN_LOCAL = 0x2 + RTN_BROADCAST = 0x3 + RTN_ANYCAST = 0x4 + RTN_MULTICAST = 0x5 + RTN_BLACKHOLE = 0x6 + RTN_UNREACHABLE = 0x7 + RTN_PROHIBIT = 0x8 + RTN_THROW = 0x9 + RTN_NAT = 0xa + RTN_XRESOLVE = 0xb + SizeofNlMsghdr = 0x10 + SizeofNlMsgerr = 0x14 + SizeofRtGenmsg = 0x1 + SizeofNlAttr = 0x4 + SizeofRtAttr = 0x4 + SizeofIfInfomsg = 0x10 + SizeofIfAddrmsg = 0x8 + SizeofRtmsg = 0xc + SizeofRtNexthop = 0x8 SizeofInotifyEvent = 0x10 ) @@ -193,6 +271,13 @@ type RawSockaddrLinklayer struct { Addr [8]uint8 } +type RawSockaddrNetlink struct { + Family uint16 + Pad uint16 + Pid uint32 + Groups uint32 +} + type RawSockaddr struct { Family uint16 Data [14]int8 @@ -215,11 +300,16 @@ type Iovec struct { Len uint64 } -type IpMreq struct { +type IPMreq struct { Multiaddr [4]byte /* in_addr */ Interface [4]byte /* in_addr */ } +type IPv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + type Msghdr struct { Name *byte Namelen uint32 @@ -238,12 +328,80 @@ type Cmsghdr struct { Type int32 } +type Inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + type Ucred struct { Pid int32 Uid uint32 Gid uint32 } +type NlMsghdr struct { + Len uint32 + Type uint16 + Flags uint16 + Seq uint32 + Pid uint32 +} + +type NlMsgerr struct { + Error int32 + Msg NlMsghdr +} + +type RtGenmsg struct { + Family uint8 +} + +type NlAttr struct { + Len uint16 + Type uint16 +} + +type RtAttr struct { + Len uint16 + Type uint16 +} + +type IfInfomsg struct { + Family uint8 + X__ifi_pad uint8 + Type uint16 + Index int32 + Flags uint32 + Change uint32 +} + +type IfAddrmsg struct { + Family uint8 + Prefixlen uint8 + Flags uint8 + Scope uint8 + Index uint32 +} + +type RtMsg struct { + Family uint8 + Dst_len uint8 + Src_len uint8 + Tos uint8 + Table uint8 + Protocol uint8 + Scope uint8 + Type uint8 + Flags uint32 +} + +type RtNexthop struct { + Len uint16 + Flags uint8 + Hops uint8 + Ifindex int32 +} + type InotifyEvent struct { Wd int32 Mask uint32 diff --git a/src/pkg/syscall/ztypes_linux_arm.go b/src/pkg/syscall/ztypes_linux_arm.go index 236155b20..ebd5379cb 100644 --- a/src/pkg/syscall/ztypes_linux_arm.go +++ b/src/pkg/syscall/ztypes_linux_arm.go @@ -5,7 +5,7 @@ // Manual corrections: TODO(rsc): need to fix godefs // remove duplicate PtraceRegs type // change RawSockaddrUnix field to Path [108]int8 (was uint8) -// add padding to EpollEvent +// add padding to EpollEvent package syscall @@ -22,11 +22,89 @@ const ( SizeofSockaddrAny = 0x70 SizeofSockaddrUnix = 0x6e SizeofSockaddrLinklayer = 0x14 + SizeofSockaddrNetlink = 0xc SizeofLinger = 0x8 - SizeofIpMreq = 0x8 + SizeofIPMreq = 0x8 + SizeofIPv6Mreq = 0x14 SizeofMsghdr = 0x1c SizeofCmsghdr = 0xc + SizeofInet6Pktinfo = 0x14 SizeofUcred = 0xc + IFA_UNSPEC = 0 + IFA_ADDRESS = 0x1 + IFA_LOCAL = 0x2 + IFA_LABEL = 0x3 + IFA_BROADCAST = 0x4 + IFA_ANYCAST = 0x5 + IFA_CACHEINFO = 0x6 + IFA_MULTICAST = 0x7 + IFLA_UNSPEC = 0 + IFLA_ADDRESS = 0x1 + IFLA_BROADCAST = 0x2 + IFLA_IFNAME = 0x3 + IFLA_MTU = 0x4 + IFLA_LINK = 0x5 + IFLA_QDISC = 0x6 + IFLA_STATS = 0x7 + IFLA_COST = 0x8 + IFLA_PRIORITY = 0x9 + IFLA_MASTER = 0xa + IFLA_WIRELESS = 0xb + IFLA_PROTINFO = 0xc + IFLA_TXQLEN = 0xd + IFLA_MAP = 0xe + IFLA_WEIGHT = 0xf + IFLA_OPERSTATE = 0x10 + IFLA_LINKMODE = 0x11 + IFLA_LINKINFO = 0x12 + IFLA_NET_NS_PID = 0x13 + IFLA_IFALIAS = 0x14 + IFLA_MAX = 0x14 + RT_SCOPE_UNIVERSE = 0 + RT_SCOPE_SITE = 0xc8 + RT_SCOPE_LINK = 0xfd + RT_SCOPE_HOST = 0xfe + RT_SCOPE_NOWHERE = 0xff + RT_TABLE_UNSPEC = 0 + RT_TABLE_COMPAT = 0xfc + RT_TABLE_DEFAULT = 0xfd + RT_TABLE_MAIN = 0xfe + RT_TABLE_LOCAL = 0xff + RT_TABLE_MAX = 0xffffffff + RTA_UNSPEC = 0 + RTA_DST = 0x1 + RTA_SRC = 0x2 + RTA_IIF = 0x3 + RTA_OIF = 0x4 + RTA_GATEWAY = 0x5 + RTA_PRIORITY = 0x6 + RTA_PREFSRC = 0x7 + RTA_METRICS = 0x8 + RTA_MULTIPATH = 0x9 + RTA_FLOW = 0xb + RTA_CACHEINFO = 0xc + RTA_TABLE = 0xf + RTN_UNSPEC = 0 + RTN_UNICAST = 0x1 + RTN_LOCAL = 0x2 + RTN_BROADCAST = 0x3 + RTN_ANYCAST = 0x4 + RTN_MULTICAST = 0x5 + RTN_BLACKHOLE = 0x6 + RTN_UNREACHABLE = 0x7 + RTN_PROHIBIT = 0x8 + RTN_THROW = 0x9 + RTN_NAT = 0xa + RTN_XRESOLVE = 0xb + SizeofNlMsghdr = 0x10 + SizeofNlMsgerr = 0x14 + SizeofRtGenmsg = 0x1 + SizeofNlAttr = 0x4 + SizeofRtAttr = 0x4 + SizeofIfInfomsg = 0x10 + SizeofIfAddrmsg = 0x8 + SizeofRtmsg = 0xc + SizeofRtNexthop = 0x8 SizeofInotifyEvent = 0x10 ) @@ -51,37 +129,37 @@ type Timeval struct { } type Timex struct { - Modes uint32 - Offset int32 - Freq int32 - Maxerror int32 - Esterror int32 - Status int32 - Constant int32 - Precision int32 - Tolerance int32 - Time Timeval - Tick int32 - Ppsfreq int32 - Jitter int32 - Shift int32 - Stabil int32 - Jitcnt int32 - Calcnt int32 - Errcnt int32 - Stbcnt int32 - Tai int32 - Pad0 int32 - Pad1 int32 - Pad2 int32 - Pad3 int32 - Pad4 int32 - Pad5 int32 - Pad6 int32 - Pad7 int32 - Pad8 int32 - Pad9 int32 - Pad10 int32 + Modes uint32 + Offset int32 + Freq int32 + Maxerror int32 + Esterror int32 + Status int32 + Constant int32 + Precision int32 + Tolerance int32 + Time Timeval + Tick int32 + Ppsfreq int32 + Jitter int32 + Shift int32 + Stabil int32 + Jitcnt int32 + Calcnt int32 + Errcnt int32 + Stbcnt int32 + Tai int32 + Pad_godefs_0 int32 + Pad_godefs_1 int32 + Pad_godefs_2 int32 + Pad_godefs_3 int32 + Pad_godefs_4 int32 + Pad_godefs_5 int32 + Pad_godefs_6 int32 + Pad_godefs_7 int32 + Pad_godefs_8 int32 + Pad_godefs_9 int32 + Pad_godefs_10 int32 } type Time_t int32 @@ -125,49 +203,49 @@ type Rlimit struct { type _Gid_t uint32 type Stat_t struct { - Dev uint64 - X__pad1 uint16 - Pad0 [2]byte - X__st_ino uint32 - Mode uint32 - Nlink uint32 - Uid uint32 - Gid uint32 - Rdev uint64 - X__pad2 uint16 - Pad1 [6]byte - Size int64 - Blksize int32 - Pad2 [4]byte - Blocks int64 - Atim Timespec - Mtim Timespec - Ctim Timespec - Ino uint64 + Dev uint64 + X__pad1 uint16 + Pad_godefs_0 [2]byte + X__st_ino uint32 + Mode uint32 + Nlink uint32 + Uid uint32 + Gid uint32 + Rdev uint64 + X__pad2 uint16 + Pad_godefs_1 [6]byte + Size int64 + Blksize int32 + Pad_godefs_2 [4]byte + Blocks int64 + Atim Timespec + Mtim Timespec + Ctim Timespec + Ino uint64 } type Statfs_t struct { - Type int32 - Bsize int32 - Blocks uint64 - Bfree uint64 - Bavail uint64 - Files uint64 - Ffree uint64 - Fsid [8]byte /* __fsid_t */ - Namelen int32 - Frsize int32 - Spare [5]int32 - Pad0 [4]byte + Type int32 + Bsize int32 + Blocks uint64 + Bfree uint64 + Bavail uint64 + Files uint64 + Ffree uint64 + Fsid [8]byte /* __fsid_t */ + Namelen int32 + Frsize int32 + Spare [5]int32 + Pad_godefs_0 [4]byte } type Dirent struct { - Ino uint64 - Off int64 - Reclen uint16 - Type uint8 - Name [256]uint8 - Pad0 [5]byte + Ino uint64 + Off int64 + Reclen uint16 + Type uint8 + Name [256]uint8 + Pad_godefs_0 [5]byte } type RawSockaddrInet4 struct { @@ -200,6 +278,13 @@ type RawSockaddrLinklayer struct { Addr [8]uint8 } +type RawSockaddrNetlink struct { + Family uint16 + Pad uint16 + Pid uint32 + Groups uint32 +} + type RawSockaddr struct { Family uint16 Data [14]uint8 @@ -222,11 +307,16 @@ type Iovec struct { Len uint32 } -type IpMreq struct { +type IPMreq struct { Multiaddr [4]byte /* in_addr */ Interface [4]byte /* in_addr */ } +type IPv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + type Msghdr struct { Name *byte Namelen uint32 @@ -243,12 +333,80 @@ type Cmsghdr struct { Type int32 } +type Inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + type Ucred struct { Pid int32 Uid uint32 Gid uint32 } +type NlMsghdr struct { + Len uint32 + Type uint16 + Flags uint16 + Seq uint32 + Pid uint32 +} + +type NlMsgerr struct { + Error int32 + Msg NlMsghdr +} + +type RtGenmsg struct { + Family uint8 +} + +type NlAttr struct { + Len uint16 + Type uint16 +} + +type RtAttr struct { + Len uint16 + Type uint16 +} + +type IfInfomsg struct { + Family uint8 + X__ifi_pad uint8 + Type uint16 + Index int32 + Flags uint32 + Change uint32 +} + +type IfAddrmsg struct { + Family uint8 + Prefixlen uint8 + Flags uint8 + Scope uint8 + Index uint32 +} + +type RtMsg struct { + Family uint8 + Dst_len uint8 + Src_len uint8 + Tos uint8 + Table uint8 + Protocol uint8 + Scope uint8 + Type uint8 + Flags uint32 +} + +type RtNexthop struct { + Len uint16 + Flags uint8 + Hops uint8 + Ifindex int32 +} + type InotifyEvent struct { Wd int32 Mask uint32 diff --git a/src/pkg/syscall/ztypes_windows_386.go b/src/pkg/syscall/ztypes_windows_386.go index 3a50be14c..b04fea576 100644 --- a/src/pkg/syscall/ztypes_windows_386.go +++ b/src/pkg/syscall/ztypes_windows_386.go @@ -49,6 +49,23 @@ const ( ) const ( + // More invented values for signals + SIGHUP = 0x1 + SIGINT = 0x2 + SIGQUIT = 0x3 + SIGILL = 0x4 + SIGTRAP = 0x5 + SIGABRT = 0x6 + SIGBUS = 0x7 + SIGFPE = 0x8 + SIGKILL = 0x9 + SIGSEGV = 0xb + SIGPIPE = 0xd + SIGALRM = 0xe + SIGTERM = 0xf +) + +const ( GENERIC_READ = 0x80000000 GENERIC_WRITE = 0x40000000 GENERIC_EXECUTE = 0x20000000 @@ -119,6 +136,18 @@ const ( STANDARD_RIGHTS_READ = 0x00020000 PROCESS_QUERY_INFORMATION = 0x00000400 SYNCHRONIZE = 0x00100000 + + PAGE_READONLY = 0x02 + PAGE_READWRITE = 0x04 + PAGE_WRITECOPY = 0x08 + PAGE_EXECUTE_READ = 0x20 + PAGE_EXECUTE_READWRITE = 0x40 + PAGE_EXECUTE_WRITECOPY = 0x80 + + FILE_MAP_COPY = 0x01 + FILE_MAP_WRITE = 0x02 + FILE_MAP_READ = 0x04 + FILE_MAP_EXECUTE = 0x20 ) const ( @@ -186,7 +215,7 @@ type Overlapped struct { InternalHigh uint32 Offset uint32 OffsetHigh uint32 - HEvent *byte + HEvent int32 } type Filetime struct { diff --git a/src/pkg/syslog/syslog_test.go b/src/pkg/syslog/syslog_test.go index 4816ddf2a..5c0b3e0c4 100644 --- a/src/pkg/syslog/syslog_test.go +++ b/src/pkg/syslog/syslog_test.go @@ -35,7 +35,19 @@ func startServer(done chan<- string) { go runSyslog(c, done) } +func skipNetTest(t *testing.T) bool { + if testing.Short() { + // Depends on syslog daemon running, and sometimes it's not. + t.Logf("skipping syslog test during -short") + return true + } + return false +} + func TestNew(t *testing.T) { + if skipNetTest(t) { + return + } s, err := New(LOG_INFO, "") if err != nil { t.Fatalf("New() failed: %s", err) @@ -45,6 +57,9 @@ func TestNew(t *testing.T) { } func TestNewLogger(t *testing.T) { + if skipNetTest(t) { + return + } f := NewLogger(LOG_INFO, 0) if f == nil { t.Error("NewLogger() failed") @@ -52,9 +67,8 @@ func TestNewLogger(t *testing.T) { } func TestDial(t *testing.T) { - if testing.Short() { - // Depends on syslog daemon running, and sometimes it's not. - t.Logf("skipping syslog test during -short") + if skipNetTest(t) { + return } l, err := Dial("", "", LOG_ERR, "syslog_test") if err != nil { diff --git a/src/pkg/tabwriter/tabwriter.go b/src/pkg/tabwriter/tabwriter.go index d91a07db2..b84c6ec79 100644 --- a/src/pkg/tabwriter/tabwriter.go +++ b/src/pkg/tabwriter/tabwriter.go @@ -158,7 +158,7 @@ const ( TabIndent // Print a vertical bar ('|') between columns (after formatting). - // Discarded colums appear as zero-width columns ("||"). + // Discarded columns appear as zero-width columns ("||"). Debug ) diff --git a/src/pkg/template/Makefile b/src/pkg/template/Makefile index 4915527b4..4f1e06527 100644 --- a/src/pkg/template/Makefile +++ b/src/pkg/template/Makefile @@ -6,7 +6,9 @@ include ../../Make.inc TARG=template GOFILES=\ + doc.go\ + execute.go\ format.go\ - template.go\ + parse.go\ include ../../Make.pkg diff --git a/src/pkg/template/doc.go b/src/pkg/template/doc.go new file mode 100644 index 000000000..e778d801d --- /dev/null +++ b/src/pkg/template/doc.go @@ -0,0 +1,91 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Package template implements data-driven templates for generating textual + output such as HTML. + + Templates are executed by applying them to a data structure. + Annotations in the template refer to elements of the data + structure (typically a field of a struct or a key in a map) + to control execution and derive values to be displayed. + The template walks the structure as it executes and the + "cursor" @ represents the value at the current location + in the structure. + + Data items may be values or pointers; the interface hides the + indirection. + + In the following, 'Field' is one of several things, according to the data. + + - The name of a field of a struct (result = data.Field), + - The value stored in a map under that key (result = data["Field"]), or + - The result of invoking a niladic single-valued method with that name + (result = data.Field()) + + If Field is a struct field or method name, it must be an exported + (capitalized) name. + + Major constructs ({} are the default delimiters for template actions; + [] are the notation in this comment for optional elements): + + {# comment } + + A one-line comment. + + {.section field} XXX [ {.or} YYY ] {.end} + + Set @ to the value of the field. It may be an explicit @ + to stay at the same point in the data. If the field is nil + or empty, execute YYY; otherwise execute XXX. + + {.repeated section field} XXX [ {.alternates with} ZZZ ] [ {.or} YYY ] {.end} + + Like .section, but field must be an array or slice. XXX + is executed for each element. If the array is nil or empty, + YYY is executed instead. If the {.alternates with} marker + is present, ZZZ is executed between iterations of XXX. + + {field} + {field1 field2 ...} + {field|formatter} + {field1 field2...|formatter} + {field|formatter1|formatter2} + + Insert the value of the fields into the output. Each field is + first looked for in the cursor, as in .section and .repeated. + If it is not found, the search continues in outer sections + until the top level is reached. + + If the field value is a pointer, leading asterisks indicate + that the value to be inserted should be evaluated through the + pointer. For example, if x.p is of type *int, {x.p} will + insert the value of the pointer but {*x.p} will insert the + value of the underlying integer. If the value is nil or not a + pointer, asterisks have no effect. + + If a formatter is specified, it must be named in the formatter + map passed to the template set up routines or in the default + set ("html","str","") and is used to process the data for + output. The formatter function has signature + func(wr io.Writer, formatter string, data ...interface{}) + where wr is the destination for output, data holds the field + values at the instantiation, and formatter is its name at + the invocation site. The default formatter just concatenates + the string representations of the fields. + + Multiple formatters separated by the pipeline character | are + executed sequentially, with each formatter receiving the bytes + emitted by the one to its left. + + As well as field names, one may use literals with Go syntax. + Integer, floating-point, and string literals are supported. + Raw strings may not span newlines. + + The delimiter strings get their default value, "{" and "}", from + JSON-template. They may be set to any non-empty, space-free + string using the SetDelims method. Their value can be printed + in the output using {.meta-left} and {.meta-right}. +*/ +package template diff --git a/src/pkg/template/execute.go b/src/pkg/template/execute.go new file mode 100644 index 000000000..5bc7ff7e9 --- /dev/null +++ b/src/pkg/template/execute.go @@ -0,0 +1,346 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code to execute a parsed template. + +package template + +import ( + "bytes" + "io" + "reflect" + "strings" +) + +// Internal state for executing a Template. As we evaluate the struct, +// the data item descends into the fields associated with sections, etc. +// Parent is used to walk upwards to find variables higher in the tree. +type state struct { + parent *state // parent in hierarchy + data reflect.Value // the driver data for this section etc. + wr io.Writer // where to send output + buf [2]bytes.Buffer // alternating buffers used when chaining formatters +} + +func (parent *state) clone(data reflect.Value) *state { + return &state{parent: parent, data: data, wr: parent.wr} +} + +// Evaluate interfaces and pointers looking for a value that can look up the name, via a +// struct field, method, or map key, and return the result of the lookup. +func (t *Template) lookup(st *state, v reflect.Value, name string) reflect.Value { + for v.IsValid() { + typ := v.Type() + if n := v.Type().NumMethod(); n > 0 { + for i := 0; i < n; i++ { + m := typ.Method(i) + mtyp := m.Type + if m.Name == name && mtyp.NumIn() == 1 && mtyp.NumOut() == 1 { + if !isExported(name) { + t.execError(st, t.linenum, "name not exported: %s in type %s", name, st.data.Type()) + } + return v.Method(i).Call(nil)[0] + } + } + } + switch av := v; av.Kind() { + case reflect.Ptr: + v = av.Elem() + case reflect.Interface: + v = av.Elem() + case reflect.Struct: + if !isExported(name) { + t.execError(st, t.linenum, "name not exported: %s in type %s", name, st.data.Type()) + } + return av.FieldByName(name) + case reflect.Map: + if v := av.MapIndex(reflect.ValueOf(name)); v.IsValid() { + return v + } + return reflect.Zero(typ.Elem()) + default: + return reflect.Value{} + } + } + return v +} + +// indirectPtr returns the item numLevels levels of indirection below the value. +// It is forgiving: if the value is not a pointer, it returns it rather than giving +// an error. If the pointer is nil, it is returned as is. +func indirectPtr(v reflect.Value, numLevels int) reflect.Value { + for i := numLevels; v.IsValid() && i > 0; i++ { + if p := v; p.Kind() == reflect.Ptr { + if p.IsNil() { + return v + } + v = p.Elem() + } else { + break + } + } + return v +} + +// Walk v through pointers and interfaces, extracting the elements within. +func indirect(v reflect.Value) reflect.Value { +loop: + for v.IsValid() { + switch av := v; av.Kind() { + case reflect.Ptr: + v = av.Elem() + case reflect.Interface: + v = av.Elem() + default: + break loop + } + } + return v +} + +// If the data for this template is a struct, find the named variable. +// Names of the form a.b.c are walked down the data tree. +// The special name "@" (the "cursor") denotes the current data. +// The value coming in (st.data) might need indirecting to reach +// a struct while the return value is not indirected - that is, +// it represents the actual named field. Leading stars indicate +// levels of indirection to be applied to the value. +func (t *Template) findVar(st *state, s string) reflect.Value { + data := st.data + flattenedName := strings.TrimLeft(s, "*") + numStars := len(s) - len(flattenedName) + s = flattenedName + if s == "@" { + return indirectPtr(data, numStars) + } + for _, elem := range strings.Split(s, ".", -1) { + // Look up field; data must be a struct or map. + data = t.lookup(st, data, elem) + if !data.IsValid() { + return reflect.Value{} + } + } + return indirectPtr(data, numStars) +} + +// Is there no data to look at? +func empty(v reflect.Value) bool { + v = indirect(v) + if !v.IsValid() { + return true + } + switch v.Kind() { + case reflect.Bool: + return v.Bool() == false + case reflect.String: + return v.String() == "" + case reflect.Struct: + return false + case reflect.Map: + return false + case reflect.Array: + return v.Len() == 0 + case reflect.Slice: + return v.Len() == 0 + } + return false +} + +// Look up a variable or method, up through the parent if necessary. +func (t *Template) varValue(name string, st *state) reflect.Value { + field := t.findVar(st, name) + if !field.IsValid() { + if st.parent == nil { + t.execError(st, t.linenum, "name not found: %s in type %s", name, st.data.Type()) + } + return t.varValue(name, st.parent) + } + return field +} + +func (t *Template) format(wr io.Writer, fmt string, val []interface{}, v *variableElement, st *state) { + fn := t.formatter(fmt) + if fn == nil { + t.execError(st, v.linenum, "missing formatter %s for variable", fmt) + } + fn(wr, fmt, val...) +} + +// Evaluate a variable, looking up through the parent if necessary. +// If it has a formatter attached ({var|formatter}) run that too. +func (t *Template) writeVariable(v *variableElement, st *state) { + // Resolve field names + val := make([]interface{}, len(v.args)) + for i, arg := range v.args { + if name, ok := arg.(fieldName); ok { + val[i] = t.varValue(string(name), st).Interface() + } else { + val[i] = arg + } + } + for i, fmt := range v.fmts[:len(v.fmts)-1] { + b := &st.buf[i&1] + b.Reset() + t.format(b, fmt, val, v, st) + val = val[0:1] + val[0] = b.Bytes() + } + t.format(st.wr, v.fmts[len(v.fmts)-1], val, v, st) +} + +// Execute element i. Return next index to execute. +func (t *Template) executeElement(i int, st *state) int { + switch elem := t.elems[i].(type) { + case *textElement: + st.wr.Write(elem.text) + return i + 1 + case *literalElement: + st.wr.Write(elem.text) + return i + 1 + case *variableElement: + t.writeVariable(elem, st) + return i + 1 + case *sectionElement: + t.executeSection(elem, st) + return elem.end + case *repeatedElement: + t.executeRepeated(elem, st) + return elem.end + } + e := t.elems[i] + t.execError(st, 0, "internal error: bad directive in execute: %v %T\n", reflect.ValueOf(e).Interface(), e) + return 0 +} + +// Execute the template. +func (t *Template) execute(start, end int, st *state) { + for i := start; i < end; { + i = t.executeElement(i, st) + } +} + +// Execute a .section +func (t *Template) executeSection(s *sectionElement, st *state) { + // Find driver data for this section. It must be in the current struct. + field := t.varValue(s.field, st) + if !field.IsValid() { + t.execError(st, s.linenum, ".section: cannot find field %s in %s", s.field, st.data.Type()) + } + st = st.clone(field) + start, end := s.start, s.or + if !empty(field) { + // Execute the normal block. + if end < 0 { + end = s.end + } + } else { + // Execute the .or block. If it's missing, do nothing. + start, end = s.or, s.end + if start < 0 { + return + } + } + for i := start; i < end; { + i = t.executeElement(i, st) + } +} + +// Return the result of calling the Iter method on v, or nil. +func iter(v reflect.Value) reflect.Value { + for j := 0; j < v.Type().NumMethod(); j++ { + mth := v.Type().Method(j) + fv := v.Method(j) + ft := fv.Type() + // TODO(rsc): NumIn() should return 0 here, because ft is from a curried FuncValue. + if mth.Name != "Iter" || ft.NumIn() != 1 || ft.NumOut() != 1 { + continue + } + ct := ft.Out(0) + if ct.Kind() != reflect.Chan || + ct.ChanDir()&reflect.RecvDir == 0 { + continue + } + return fv.Call(nil)[0] + } + return reflect.Value{} +} + +// Execute a .repeated section +func (t *Template) executeRepeated(r *repeatedElement, st *state) { + // Find driver data for this section. It must be in the current struct. + field := t.varValue(r.field, st) + if !field.IsValid() { + t.execError(st, r.linenum, ".repeated: cannot find field %s in %s", r.field, st.data.Type()) + } + field = indirect(field) + + start, end := r.start, r.or + if end < 0 { + end = r.end + } + if r.altstart >= 0 { + end = r.altstart + } + first := true + + // Code common to all the loops. + loopBody := func(newst *state) { + // .alternates between elements + if !first && r.altstart >= 0 { + for i := r.altstart; i < r.altend; { + i = t.executeElement(i, newst) + } + } + first = false + for i := start; i < end; { + i = t.executeElement(i, newst) + } + } + + if array := field; array.Kind() == reflect.Array || array.Kind() == reflect.Slice { + for j := 0; j < array.Len(); j++ { + loopBody(st.clone(array.Index(j))) + } + } else if m := field; m.Kind() == reflect.Map { + for _, key := range m.MapKeys() { + loopBody(st.clone(m.MapIndex(key))) + } + } else if ch := iter(field); ch.IsValid() { + for { + e, ok := ch.Recv() + if !ok { + break + } + loopBody(st.clone(e)) + } + } else { + t.execError(st, r.linenum, ".repeated: cannot repeat %s (type %s)", + r.field, field.Type()) + } + + if first { + // Empty. Execute the .or block, once. If it's missing, do nothing. + start, end := r.or, r.end + if start >= 0 { + newst := st.clone(field) + for i := start; i < end; { + i = t.executeElement(i, newst) + } + } + return + } +} + +// A valid delimiter must contain no space and be non-empty. +func validDelim(d []byte) bool { + if len(d) == 0 { + return false + } + for _, c := range d { + if isSpace(c) { + return false + } + } + return true +} diff --git a/src/pkg/template/template.go b/src/pkg/template/parse.go index 253207852..b4aa5fcd2 100644 --- a/src/pkg/template/template.go +++ b/src/pkg/template/parse.go @@ -2,95 +2,17 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -/* - Package template implements data-driven templates for generating textual - output such as HTML. +// Code to parse a template. - Templates are executed by applying them to a data structure. - Annotations in the template refer to elements of the data - structure (typically a field of a struct or a key in a map) - to control execution and derive values to be displayed. - The template walks the structure as it executes and the - "cursor" @ represents the value at the current location - in the structure. - - Data items may be values or pointers; the interface hides the - indirection. - - In the following, 'field' is one of several things, according to the data. - - - The name of a field of a struct (result = data.field), - - The value stored in a map under that key (result = data[field]), or - - The result of invoking a niladic single-valued method with that name - (result = data.field()) - - Major constructs ({} are the default delimiters for template actions; - [] are the notation in this comment for optional elements): - - {# comment } - - A one-line comment. - - {.section field} XXX [ {.or} YYY ] {.end} - - Set @ to the value of the field. It may be an explicit @ - to stay at the same point in the data. If the field is nil - or empty, execute YYY; otherwise execute XXX. - - {.repeated section field} XXX [ {.alternates with} ZZZ ] [ {.or} YYY ] {.end} - - Like .section, but field must be an array or slice. XXX - is executed for each element. If the array is nil or empty, - YYY is executed instead. If the {.alternates with} marker - is present, ZZZ is executed between iterations of XXX. - - {field} - {field1 field2 ...} - {field|formatter} - {field1 field2...|formatter} - {field|formatter1|formatter2} - - Insert the value of the fields into the output. Each field is - first looked for in the cursor, as in .section and .repeated. - If it is not found, the search continues in outer sections - until the top level is reached. - - If the field value is a pointer, leading asterisks indicate - that the value to be inserted should be evaluated through the - pointer. For example, if x.p is of type *int, {x.p} will - insert the value of the pointer but {*x.p} will insert the - value of the underlying integer. If the value is nil or not a - pointer, asterisks have no effect. - - If a formatter is specified, it must be named in the formatter - map passed to the template set up routines or in the default - set ("html","str","") and is used to process the data for - output. The formatter function has signature - func(wr io.Writer, formatter string, data ...interface{}) - where wr is the destination for output, data holds the field - values at the instantiation, and formatter is its name at - the invocation site. The default formatter just concatenates - the string representations of the fields. - - Multiple formatters separated by the pipeline character | are - executed sequentially, with each formatter receiving the bytes - emitted by the one to its left. - - The delimiter strings get their default value, "{" and "}", from - JSON-template. They may be set to any non-empty, space-free - string using the SetDelims method. Their value can be printed - in the output using {.meta-left} and {.meta-right}. -*/ package template import ( - "bytes" - "container/vector" "fmt" "io" "io/ioutil" "os" "reflect" + "strconv" "strings" "unicode" "utf8" @@ -105,6 +27,19 @@ type Error struct { func (e *Error) String() string { return fmt.Sprintf("line %d: %s", e.Line, e.Msg) } +// checkError is a deferred function to turn a panic with type *Error into a plain error return. +// Other panics are unexpected and so are re-enabled. +func checkError(error *os.Error) { + if v := recover(); v != nil { + if e, ok := v.(*Error); ok { + *error = e + } else { + // runtime errors should crash + panic(v) + } + } +} + // Most of the literals are aces. var lbrace = []byte{'{'} var rbrace = []byte{'}'} @@ -151,10 +86,13 @@ type literalElement struct { // A variable invocation to be evaluated type variableElement struct { linenum int - word []string // The fields in the invocation. - fmts []string // Names of formatters to apply. len(fmts) > 0 + args []interface{} // The fields and literals in the invocation. + fmts []string // Names of formatters to apply. len(fmts) > 0 } +// A variableElement arg to be evaluated as a field name +type fieldName string + // A .section block, possibly with a .or type sectionElement struct { linenum int // of .section itself @@ -181,21 +119,7 @@ type Template struct { p int // position in buf linenum int // position in input // Parsed results: - elems *vector.Vector -} - -// Internal state for executing a Template. As we evaluate the struct, -// the data item descends into the fields associated with sections, etc. -// Parent is used to walk upwards to find variables higher in the tree. -type state struct { - parent *state // parent in hierarchy - data reflect.Value // the driver data for this section etc. - wr io.Writer // where to send output - buf [2]bytes.Buffer // alternating buffers used when chaining formatters -} - -func (parent *state) clone(data reflect.Value) *state { - return &state{parent: parent, data: data, wr: parent.wr} + elems []interface{} } // New creates a new template with the specified formatter map (which @@ -205,7 +129,7 @@ func New(fmap FormatterMap) *Template { t.fmap = fmap t.ldelim = lbrace t.rdelim = rbrace - t.elems = new(vector.Vector) + t.elems = make([]interface{}, 0, 16) return t } @@ -228,8 +152,8 @@ func isExported(name string) bool { // -- Lexical analysis -// Is c a white space character? -func white(c uint8) bool { return c == ' ' || c == '\t' || c == '\r' || c == '\n' } +// Is c a space character? +func isSpace(c uint8) bool { return c == ' ' || c == '\t' || c == '\r' || c == '\n' } // Safely, does s[n:n+len(t)] == t? func equal(s []byte, n int, t []byte) bool { @@ -245,6 +169,31 @@ func equal(s []byte, n int, t []byte) bool { return true } +// isQuote returns true if c is a string- or character-delimiting quote character. +func isQuote(c byte) bool { + return c == '"' || c == '`' || c == '\'' +} + +// endQuote returns the end quote index for the quoted string that +// starts at n, or -1 if no matching end quote is found before the end +// of the line. +func endQuote(s []byte, n int) int { + quote := s[n] + for n++; n < len(s); n++ { + switch s[n] { + case '\\': + if quote == '"' || quote == '\'' { + n++ + } + case '\n': + return -1 + case quote: + return n + } + } + return -1 +} + // nextItem returns the next item from the input buffer. If the returned // item is empty, we are at EOF. The item will be either a // delimited string or a non-empty string between delimited @@ -259,9 +208,9 @@ func (t *Template) nextItem() []byte { t.linenum++ i++ } - // Leading white space up to but not including newline + // Leading space up to but not including newline for i = start; i < len(t.buf); i++ { - if t.buf[i] == '\n' || !white(t.buf[i]) { + if t.buf[i] == '\n' || !isSpace(t.buf[i]) { break } } @@ -282,6 +231,14 @@ func (t *Template) nextItem() []byte { if t.buf[i] == '\n' { break } + if isQuote(t.buf[i]) { + i = endQuote(t.buf, i) + if i == -1 { + t.parseError("unmatched quote") + return nil + } + continue + } if equal(t.buf, i, t.rdelim) { i += len(t.rdelim) right = i @@ -298,7 +255,7 @@ func (t *Template) nextItem() []byte { firstChar := t.buf[left+len(t.ldelim)] if firstChar == '.' || firstChar == '#' { // It's special and the first thing on the line. Is it the last? - for j := right; j < len(t.buf) && white(t.buf[j]); j++ { + for j := right; j < len(t.buf) && isSpace(t.buf[j]); j++ { if t.buf[j] == '\n' { // Yes it is. Drop the surrounding space and return the {.foo} t.linenum++ @@ -310,7 +267,7 @@ func (t *Template) nextItem() []byte { } // No it's not. If there's leading space, return that. if leadingSpace { - // not trimming space: return leading white space if there is some. + // not trimming space: return leading space if there is some. t.p = left return t.buf[start:left] } @@ -333,23 +290,34 @@ func (t *Template) nextItem() []byte { return item } -// Turn a byte array into a white-space-split array of strings. +// Turn a byte array into a space-split array of strings, +// taking into account quoted strings. func words(buf []byte) []string { s := make([]string, 0, 5) - p := 0 // position in buf - // one word per loop - for i := 0; ; i++ { - // skip white space - for ; p < len(buf) && white(buf[p]); p++ { - } - // grab word - start := p - for ; p < len(buf) && !white(buf[p]); p++ { + for i := 0; i < len(buf); { + // One word per loop + for i < len(buf) && isSpace(buf[i]) { + i++ } - if start == p { // no text left + if i == len(buf) { break } - s = append(s, string(buf[start:p])) + // Got a word + start := i + if isQuote(buf[i]) { + i = endQuote(buf, i) + if i < 0 { + i = len(buf) + } else { + i++ + } + } + // Even with quotes, break on space only. This handles input + // such as {""|} and catches quoting mistakes. + for i < len(buf) && !isSpace(buf[i]) { + i++ + } + s = append(s, string(buf[start:i])) } return s } @@ -381,11 +349,17 @@ func (t *Template) analyze(item []byte) (tok int, w []string) { t.parseError("empty directive") return } - if len(w) > 0 && w[0][0] != '.' { + first := w[0] + if first[0] != '.' { + tok = tokVariable + return + } + if len(first) > 1 && first[1] >= '0' && first[1] <= '9' { + // Must be a float. tok = tokVariable return } - switch w[0] { + switch first { case ".meta-left", ".meta-right", ".space", ".tab": tok = tokLiteral return @@ -433,18 +407,38 @@ func (t *Template) formatter(name string) func(io.Writer, string, ...interface{} // -- Parsing -// Allocate a new variable-evaluation element. +// newVariable allocates a new variable-evaluation element. func (t *Template) newVariable(words []string) *variableElement { - // After the final space-separated argument, formatters may be specified separated - // by pipe symbols, for example: {a b c|d|e} + formatters := extractFormatters(words) + args := make([]interface{}, len(words)) + + // Build argument list, processing any literals + for i, word := range words { + var lerr os.Error + switch word[0] { + case '"', '`', '\'': + v, err := strconv.Unquote(word) + if err == nil && word[0] == '\'' { + args[i] = []int(v)[0] + } else { + args[i], lerr = v, err + } - // Until we learn otherwise, formatters contains a single name: "", the default formatter. - formatters := []string{""} - lastWord := words[len(words)-1] - bar := strings.IndexRune(lastWord, '|') - if bar >= 0 { - words[len(words)-1] = lastWord[0:bar] - formatters = strings.Split(lastWord[bar+1:], "|", -1) + case '.', '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + v, err := strconv.Btoi64(word, 0) + if err == nil { + args[i] = v + } else { + v, err := strconv.Atof64(word) + args[i], lerr = v, err + } + + default: + args[i] = fieldName(word) + } + if lerr != nil { + t.parseError("invalid literal: %q: %s", word, lerr) + } } // We could remember the function address here and avoid the lookup later, @@ -457,7 +451,40 @@ func (t *Template) newVariable(words []string) *variableElement { t.parseError("unknown formatter: %q", f) } } - return &variableElement{t.linenum, words, formatters} + + return &variableElement{t.linenum, args, formatters} +} + +// extractFormatters extracts a list of formatters from words. +// After the final space-separated argument in a variable, formatters may be +// specified separated by pipe symbols. For example: {a b c|d|e} +// The words parameter still has the formatters joined by '|' in the last word. +// extractFormatters splits formatters, replaces the last word with the content +// found before the first '|' within it, and returns the formatters obtained. +// If no formatters are found in words, the default formatter is returned. +func extractFormatters(words []string) (formatters []string) { + // "" is the default formatter. + formatters = []string{""} + if len(words) == 0 { + return + } + var bar int + lastWord := words[len(words)-1] + if isQuote(lastWord[0]) { + end := endQuote([]byte(lastWord), 0) + if end < 0 || end+1 == len(lastWord) || lastWord[end+1] != '|' { + return + } + bar = end + 1 + } else { + bar = strings.IndexRune(lastWord, '|') + if bar < 0 { + return + } + } + words[len(words)-1] = lastWord[0:bar] + formatters = strings.Split(lastWord[bar+1:], "|", -1) + return } // Grab the next item. If it's simple, just append it to the template. @@ -469,24 +496,24 @@ func (t *Template) parseSimple(item []byte) (done bool, tok int, w []string) { case tokComment: return case tokText: - t.elems.Push(&textElement{item}) + t.elems = append(t.elems, &textElement{item}) return case tokLiteral: switch w[0] { case ".meta-left": - t.elems.Push(&literalElement{t.ldelim}) + t.elems = append(t.elems, &literalElement{t.ldelim}) case ".meta-right": - t.elems.Push(&literalElement{t.rdelim}) + t.elems = append(t.elems, &literalElement{t.rdelim}) case ".space": - t.elems.Push(&literalElement{space}) + t.elems = append(t.elems, &literalElement{space}) case ".tab": - t.elems.Push(&literalElement{tab}) + t.elems = append(t.elems, &literalElement{tab}) default: t.parseError("internal error: unknown literal: %s", w[0]) } return case tokVariable: - t.elems.Push(t.newVariable(w)) + t.elems = append(t.elems, t.newVariable(w)) return } return false, tok, w @@ -496,11 +523,11 @@ func (t *Template) parseSimple(item []byte) (done bool, tok int, w []string) { func (t *Template) parseRepeated(words []string) *repeatedElement { r := new(repeatedElement) - t.elems.Push(r) + t.elems = append(t.elems, r) r.linenum = t.linenum r.field = words[2] // Scan section, collecting true and false (.or) blocks. - r.start = t.elems.Len() + r.start = len(t.elems) r.or = -1 r.altstart = -1 r.altend = -1 @@ -523,8 +550,8 @@ Loop: t.parseError("extra .or in .repeated section") break Loop } - r.altend = t.elems.Len() - r.or = t.elems.Len() + r.altend = len(t.elems) + r.or = len(t.elems) case tokSection: t.parseSection(w) case tokRepeated: @@ -538,26 +565,26 @@ Loop: t.parseError(".alternates inside .or block in .repeated section") break Loop } - r.altstart = t.elems.Len() + r.altstart = len(t.elems) default: t.parseError("internal error: unknown repeated section item: %s", item) break Loop } } if r.altend < 0 { - r.altend = t.elems.Len() + r.altend = len(t.elems) } - r.end = t.elems.Len() + r.end = len(t.elems) return r } func (t *Template) parseSection(words []string) *sectionElement { s := new(sectionElement) - t.elems.Push(s) + t.elems = append(t.elems, s) s.linenum = t.linenum s.field = words[1] // Scan section, collecting true and false (.or) blocks. - s.start = t.elems.Len() + s.start = len(t.elems) s.or = -1 Loop: for { @@ -578,7 +605,7 @@ Loop: t.parseError("extra .or in .section") break Loop } - s.or = t.elems.Len() + s.or = len(t.elems) case tokSection: t.parseSection(w) case tokRepeated: @@ -589,7 +616,7 @@ Loop: t.parseError("internal error: unknown section item: %s", item) } } - s.end = t.elems.Len() + s.end = len(t.elems) return s } @@ -618,334 +645,6 @@ func (t *Template) parse() { // -- Execution -// Evaluate interfaces and pointers looking for a value that can look up the name, via a -// struct field, method, or map key, and return the result of the lookup. -func (t *Template) lookup(st *state, v reflect.Value, name string) reflect.Value { - for v.IsValid() { - typ := v.Type() - if n := v.Type().NumMethod(); n > 0 { - for i := 0; i < n; i++ { - m := typ.Method(i) - mtyp := m.Type - if m.Name == name && mtyp.NumIn() == 1 && mtyp.NumOut() == 1 { - if !isExported(name) { - t.execError(st, t.linenum, "name not exported: %s in type %s", name, st.data.Type()) - } - return v.Method(i).Call(nil)[0] - } - } - } - switch av := v; av.Kind() { - case reflect.Ptr: - v = av.Elem() - case reflect.Interface: - v = av.Elem() - case reflect.Struct: - if !isExported(name) { - t.execError(st, t.linenum, "name not exported: %s in type %s", name, st.data.Type()) - } - return av.FieldByName(name) - case reflect.Map: - if v := av.MapIndex(reflect.ValueOf(name)); v.IsValid() { - return v - } - return reflect.Zero(typ.Elem()) - default: - return reflect.Value{} - } - } - return v -} - -// indirectPtr returns the item numLevels levels of indirection below the value. -// It is forgiving: if the value is not a pointer, it returns it rather than giving -// an error. If the pointer is nil, it is returned as is. -func indirectPtr(v reflect.Value, numLevels int) reflect.Value { - for i := numLevels; v.IsValid() && i > 0; i++ { - if p := v; p.Kind() == reflect.Ptr { - if p.IsNil() { - return v - } - v = p.Elem() - } else { - break - } - } - return v -} - -// Walk v through pointers and interfaces, extracting the elements within. -func indirect(v reflect.Value) reflect.Value { -loop: - for v.IsValid() { - switch av := v; av.Kind() { - case reflect.Ptr: - v = av.Elem() - case reflect.Interface: - v = av.Elem() - default: - break loop - } - } - return v -} - -// If the data for this template is a struct, find the named variable. -// Names of the form a.b.c are walked down the data tree. -// The special name "@" (the "cursor") denotes the current data. -// The value coming in (st.data) might need indirecting to reach -// a struct while the return value is not indirected - that is, -// it represents the actual named field. Leading stars indicate -// levels of indirection to be applied to the value. -func (t *Template) findVar(st *state, s string) reflect.Value { - data := st.data - flattenedName := strings.TrimLeft(s, "*") - numStars := len(s) - len(flattenedName) - s = flattenedName - if s == "@" { - return indirectPtr(data, numStars) - } - for _, elem := range strings.Split(s, ".", -1) { - // Look up field; data must be a struct or map. - data = t.lookup(st, data, elem) - if !data.IsValid() { - return reflect.Value{} - } - } - return indirectPtr(data, numStars) -} - -// Is there no data to look at? -func empty(v reflect.Value) bool { - v = indirect(v) - if !v.IsValid() { - return true - } - switch v.Kind() { - case reflect.Bool: - return v.Bool() == false - case reflect.String: - return v.String() == "" - case reflect.Struct: - return false - case reflect.Map: - return false - case reflect.Array: - return v.Len() == 0 - case reflect.Slice: - return v.Len() == 0 - } - return false -} - -// Look up a variable or method, up through the parent if necessary. -func (t *Template) varValue(name string, st *state) reflect.Value { - field := t.findVar(st, name) - if !field.IsValid() { - if st.parent == nil { - t.execError(st, t.linenum, "name not found: %s in type %s", name, st.data.Type()) - } - return t.varValue(name, st.parent) - } - return field -} - -func (t *Template) format(wr io.Writer, fmt string, val []interface{}, v *variableElement, st *state) { - fn := t.formatter(fmt) - if fn == nil { - t.execError(st, v.linenum, "missing formatter %s for variable %s", fmt, v.word[0]) - } - fn(wr, fmt, val...) -} - -// Evaluate a variable, looking up through the parent if necessary. -// If it has a formatter attached ({var|formatter}) run that too. -func (t *Template) writeVariable(v *variableElement, st *state) { - // Turn the words of the invocation into values. - val := make([]interface{}, len(v.word)) - for i, word := range v.word { - val[i] = t.varValue(word, st).Interface() - } - - for i, fmt := range v.fmts[:len(v.fmts)-1] { - b := &st.buf[i&1] - b.Reset() - t.format(b, fmt, val, v, st) - val = val[0:1] - val[0] = b.Bytes() - } - t.format(st.wr, v.fmts[len(v.fmts)-1], val, v, st) -} - -// Execute element i. Return next index to execute. -func (t *Template) executeElement(i int, st *state) int { - switch elem := t.elems.At(i).(type) { - case *textElement: - st.wr.Write(elem.text) - return i + 1 - case *literalElement: - st.wr.Write(elem.text) - return i + 1 - case *variableElement: - t.writeVariable(elem, st) - return i + 1 - case *sectionElement: - t.executeSection(elem, st) - return elem.end - case *repeatedElement: - t.executeRepeated(elem, st) - return elem.end - } - e := t.elems.At(i) - t.execError(st, 0, "internal error: bad directive in execute: %v %T\n", reflect.ValueOf(e).Interface(), e) - return 0 -} - -// Execute the template. -func (t *Template) execute(start, end int, st *state) { - for i := start; i < end; { - i = t.executeElement(i, st) - } -} - -// Execute a .section -func (t *Template) executeSection(s *sectionElement, st *state) { - // Find driver data for this section. It must be in the current struct. - field := t.varValue(s.field, st) - if !field.IsValid() { - t.execError(st, s.linenum, ".section: cannot find field %s in %s", s.field, st.data.Type()) - } - st = st.clone(field) - start, end := s.start, s.or - if !empty(field) { - // Execute the normal block. - if end < 0 { - end = s.end - } - } else { - // Execute the .or block. If it's missing, do nothing. - start, end = s.or, s.end - if start < 0 { - return - } - } - for i := start; i < end; { - i = t.executeElement(i, st) - } -} - -// Return the result of calling the Iter method on v, or nil. -func iter(v reflect.Value) reflect.Value { - for j := 0; j < v.Type().NumMethod(); j++ { - mth := v.Type().Method(j) - fv := v.Method(j) - ft := fv.Type() - // TODO(rsc): NumIn() should return 0 here, because ft is from a curried FuncValue. - if mth.Name != "Iter" || ft.NumIn() != 1 || ft.NumOut() != 1 { - continue - } - ct := ft.Out(0) - if ct.Kind() != reflect.Chan || - ct.ChanDir()&reflect.RecvDir == 0 { - continue - } - return fv.Call(nil)[0] - } - return reflect.Value{} -} - -// Execute a .repeated section -func (t *Template) executeRepeated(r *repeatedElement, st *state) { - // Find driver data for this section. It must be in the current struct. - field := t.varValue(r.field, st) - if !field.IsValid() { - t.execError(st, r.linenum, ".repeated: cannot find field %s in %s", r.field, st.data.Type()) - } - field = indirect(field) - - start, end := r.start, r.or - if end < 0 { - end = r.end - } - if r.altstart >= 0 { - end = r.altstart - } - first := true - - // Code common to all the loops. - loopBody := func(newst *state) { - // .alternates between elements - if !first && r.altstart >= 0 { - for i := r.altstart; i < r.altend; { - i = t.executeElement(i, newst) - } - } - first = false - for i := start; i < end; { - i = t.executeElement(i, newst) - } - } - - if array := field; array.Kind() == reflect.Array || array.Kind() == reflect.Slice { - for j := 0; j < array.Len(); j++ { - loopBody(st.clone(array.Index(j))) - } - } else if m := field; m.Kind() == reflect.Map { - for _, key := range m.MapKeys() { - loopBody(st.clone(m.MapIndex(key))) - } - } else if ch := iter(field); ch.IsValid() { - for { - e, ok := ch.Recv() - if !ok { - break - } - loopBody(st.clone(e)) - } - } else { - t.execError(st, r.linenum, ".repeated: cannot repeat %s (type %s)", - r.field, field.Type()) - } - - if first { - // Empty. Execute the .or block, once. If it's missing, do nothing. - start, end := r.or, r.end - if start >= 0 { - newst := st.clone(field) - for i := start; i < end; { - i = t.executeElement(i, newst) - } - } - return - } -} - -// A valid delimiter must contain no white space and be non-empty. -func validDelim(d []byte) bool { - if len(d) == 0 { - return false - } - for _, c := range d { - if white(c) { - return false - } - } - return true -} - -// checkError is a deferred function to turn a panic with type *Error into a plain error return. -// Other panics are unexpected and so are re-enabled. -func checkError(error *os.Error) { - if v := recover(); v != nil { - if e, ok := v.(*Error); ok { - *error = e - } else { - // runtime errors should crash - panic(v) - } - } -} - // -- Public interface // Parse initializes a Template by parsing its definition. The string @@ -983,7 +682,7 @@ func (t *Template) Execute(wr io.Writer, data interface{}) (err os.Error) { val := reflect.ValueOf(data) defer checkError(&err) t.p = 0 - t.execute(0, t.elems.Len(), &state{parent: nil, data: val, wr: wr}) + t.execute(0, len(t.elems), &state{parent: nil, data: val, wr: wr}) return nil } diff --git a/src/pkg/template/template_test.go b/src/pkg/template/template_test.go index d21a5397a..99b23c288 100644 --- a/src/pkg/template/template_test.go +++ b/src/pkg/template/template_test.go @@ -94,10 +94,15 @@ func multiword(w io.Writer, format string, value ...interface{}) { } } +func printf(w io.Writer, format string, v ...interface{}) { + io.WriteString(w, fmt.Sprintf(v[0].(string), v[1:]...)) +} + var formatters = FormatterMap{ "uppercase": writer(uppercase), "+1": writer(plus1), "multiword": multiword, + "printf": printf, } var tests = []*Test{ @@ -138,6 +143,36 @@ var tests = []*Test{ out: "nil pointer: <nil>=77\n", }, + &Test{ + in: `{"Strings" ":"} {""} {"|"} {"\t\u0123 \x23\\"} {"\"}{\\"}`, + + out: "Strings: | \t\u0123 \x23\\ \"}{\\", + }, + + &Test{ + in: "{`Raw strings` `:`} {``} {`|`} {`\\t\\u0123 \\x23\\`} {`}{\\`}", + + out: "Raw strings: | \\t\\u0123 \\x23\\ }{\\", + }, + + &Test{ + in: "Characters: {'a'} {'\\u0123'} {' '} {'{'} {'|'} {'}'}", + + out: "Characters: 97 291 32 123 124 125", + }, + + &Test{ + in: "Integers: {1} {-2} {+42} {0777} {0x0a}", + + out: "Integers: 1 -2 42 511 10", + }, + + &Test{ + in: "Floats: {.5} {-.5} {1.1} {-2.2} {+42.1} {1e10} {1.2e-3} {1.2e3} {-1.2e3}", + + out: "Floats: 0.5 -0.5 1.1 -2.2 42.1 1e+10 0.0012 1200 -1200", + }, + // Method at top level &Test{ in: "ptrmethod={PointerMethod}\n", @@ -723,6 +758,14 @@ var formatterTests = []Test{ in: "{Integer|||||}", // empty string is a valid formatter out: "77", }, + { + in: `{"%.02f 0x%02X" 1.1 10|printf}`, + out: "1.10 0x0A", + }, + { + in: `{""|}{""||}{""|printf}`, // Issue #1896. + out: "", + }, } func TestFormatters(t *testing.T) { diff --git a/src/pkg/testing/benchmark.go b/src/pkg/testing/benchmark.go index cf73e2b48..f8b53e63a 100644 --- a/src/pkg/testing/benchmark.go +++ b/src/pkg/testing/benchmark.go @@ -143,14 +143,13 @@ func (b *B) run() BenchmarkResult { b.runN(n) } return BenchmarkResult{b.N, b.ns, b.bytes} - } // The results of a benchmark run. type BenchmarkResult struct { N int // The number of iterations. Ns int64 // The total time taken. - Bytes int64 // The total number of bytes processed. + Bytes int64 // Bytes processed in one iteration. } func (r BenchmarkResult) NsPerOp() int64 { @@ -160,13 +159,20 @@ func (r BenchmarkResult) NsPerOp() int64 { return r.Ns / int64(r.N) } +func (r BenchmarkResult) mbPerSec() float64 { + if r.Bytes <= 0 || r.Ns <= 0 || r.N <= 0 { + return 0 + } + return float64(r.Bytes) * float64(r.N) / float64(r.Ns) * 1e3 +} + func (r BenchmarkResult) String() string { - ns := r.NsPerOp() + mbs := r.mbPerSec() mb := "" - if ns > 0 && r.Bytes > 0 { - mb = fmt.Sprintf("\t%7.2f MB/s", (float64(r.Bytes)/1e6)/(float64(ns)/1e9)) + if mbs != 0 { + mb = fmt.Sprintf("\t%7.2f MB/s", mbs) } - return fmt.Sprintf("%8d\t%10d ns/op%s", r.N, ns, mb) + return fmt.Sprintf("%8d\t%10d ns/op%s", r.N, r.NsPerOp(), mb) } // An internal function but exported because it is cross-package; part of the implementation @@ -176,6 +182,7 @@ func RunBenchmarks(matchString func(pat, str string) (bool, os.Error), benchmark if len(*matchBenchmarks) == 0 { return } + procs := runtime.GOMAXPROCS(-1) for _, Benchmark := range benchmarks { matched, err := matchString(*matchBenchmarks, Benchmark.Name) if err != nil { @@ -187,7 +194,12 @@ func RunBenchmarks(matchString func(pat, str string) (bool, os.Error), benchmark } b := &B{benchmark: Benchmark} r := b.run() - fmt.Printf("%s\t%v\n", Benchmark.Name, r) + print(fmt.Sprintf("%s\t%v\n", Benchmark.Name, r)) + if p := runtime.GOMAXPROCS(-1); p != procs { + print(fmt.Sprintf("%s left GOMAXPROCS set to %d\n", Benchmark.Name, p)) + procs = p + } + } } diff --git a/src/pkg/testing/testing.go b/src/pkg/testing/testing.go index 8781b207d..3b2dd377a 100644 --- a/src/pkg/testing/testing.go +++ b/src/pkg/testing/testing.go @@ -171,6 +171,7 @@ func RunTests(matchString func(pat, str string) (bool, os.Error), tests []Intern if len(tests) == 0 { println("testing: warning: no tests to run") } + procs := runtime.GOMAXPROCS(-1) for i := 0; i < len(tests); i++ { matched, err := matchString(*match, tests[i].Name) if err != nil { @@ -190,6 +191,11 @@ func RunTests(matchString func(pat, str string) (bool, os.Error), tests []Intern <-t.ch ns += time.Nanoseconds() tstr := fmt.Sprintf("(%.2f seconds)", float64(ns)/1e9) + if p := runtime.GOMAXPROCS(-1); t.failed == false && p != procs { + t.failed = true + t.errors = fmt.Sprintf("%s left GOMAXPROCS set to %d\n", tests[i].Name, p) + procs = p + } if t.failed { println("--- FAIL:", tests[i].Name, tstr) print(t.errors) diff --git a/src/pkg/time/format.go b/src/pkg/time/format.go index 7b5a8f3b6..47d736342 100644 --- a/src/pkg/time/format.go +++ b/src/pkg/time/format.go @@ -272,9 +272,19 @@ func (t *Time) Format(layout string) string { case stdHour: p = zeroPad(t.Hour) case stdHour12: - p = strconv.Itoa(t.Hour % 12) + // Noon is 12PM, midnight is 12AM. + hr := t.Hour % 12 + if hr == 0 { + hr = 12 + } + p = strconv.Itoa(hr) case stdZeroHour12: - p = zeroPad(t.Hour % 12) + // Noon is 12PM, midnight is 12AM. + hr := t.Hour % 12 + if hr == 0 { + hr = 12 + } + p = zeroPad(hr) case stdMinute: p = strconv.Itoa(t.Minute) case stdZeroMinute: @@ -429,6 +439,7 @@ func skip(value, prefix string) (string, os.Error) { func Parse(alayout, avalue string) (*Time, os.Error) { var t Time rangeErrString := "" // set if a value is out of range + amSet := false // do we need to subtract 12 from the hour for midnight? pmSet := false // do we need to add 12 to the hour? layout, value := alayout, avalue // Each iteration processes one std value. @@ -558,9 +569,12 @@ func Parse(alayout, avalue string) (*Time, os.Error) { break } p, value = value[0:2], value[2:] - if p == "PM" { + switch p { + case "PM": pmSet = true - } else if p != "AM" { + case "AM": + amSet = true + default: err = errBad } case stdpm: @@ -569,9 +583,12 @@ func Parse(alayout, avalue string) (*Time, os.Error) { break } p, value = value[0:2], value[2:] - if p == "pm" { + switch p { + case "pm": pmSet = true - } else if p != "am" { + case "am": + amSet = true + default: err = errBad } case stdTZ: @@ -613,6 +630,8 @@ func Parse(alayout, avalue string) (*Time, os.Error) { } if pmSet && t.Hour < 12 { t.Hour += 12 + } else if amSet && t.Hour == 12 { + t.Hour = 0 } return &t, nil } diff --git a/src/pkg/time/sleep.go b/src/pkg/time/sleep.go index 3bc253c94..314622d0d 100644 --- a/src/pkg/time/sleep.go +++ b/src/pkg/time/sleep.go @@ -91,7 +91,7 @@ func (e *Timer) Stop() (ok bool) { // It assumes that f will not block. func after(ns int64, f func(int64)) (e *Timer) { now := Nanoseconds() - t := Nanoseconds() + ns + t := now + ns if ns > 0 && t < now { panic("time: time overflow") } diff --git a/src/pkg/time/tick.go b/src/pkg/time/tick.go index 6c21bf19b..dde18000d 100644 --- a/src/pkg/time/tick.go +++ b/src/pkg/time/tick.go @@ -88,7 +88,7 @@ func wakeLoop(wakeMeAt chan int64, wakeUp chan bool) { // A single tickerLoop serves all ticks to Tickers. It waits for two events: // either the creation of a new Ticker or a tick from the alarm, -// signalling a time to wake up one or more Tickers. +// signaling a time to wake up one or more Tickers. func tickerLoop() { // Represents the next alarm to be delivered. var alarm alarmer diff --git a/src/pkg/time/time_test.go b/src/pkg/time/time_test.go index 1d83291c0..eb676bf64 100644 --- a/src/pkg/time/time_test.go +++ b/src/pkg/time/time_test.go @@ -302,6 +302,66 @@ func TestParseErrors(t *testing.T) { } } +func TestNoonIs12PM(t *testing.T) { + noon := Time{Hour: 12} + const expect = "12:00PM" + got := noon.Format("3:04PM") + if got != expect { + t.Errorf("got %q; expect %q", got, expect) + } + got = noon.Format("03:04PM") + if got != expect { + t.Errorf("got %q; expect %q", got, expect) + } +} + +func TestMidnightIs12AM(t *testing.T) { + midnight := Time{Hour: 0} + expect := "12:00AM" + got := midnight.Format("3:04PM") + if got != expect { + t.Errorf("got %q; expect %q", got, expect) + } + got = midnight.Format("03:04PM") + if got != expect { + t.Errorf("got %q; expect %q", got, expect) + } +} + +func Test12PMIsNoon(t *testing.T) { + noon, err := Parse("3:04PM", "12:00PM") + if err != nil { + t.Fatal("error parsing date:", err) + } + if noon.Hour != 12 { + t.Errorf("got %d; expect 12", noon.Hour) + } + noon, err = Parse("03:04PM", "12:00PM") + if err != nil { + t.Fatal("error parsing date:", err) + } + if noon.Hour != 12 { + t.Errorf("got %d; expect 12", noon.Hour) + } +} + +func Test12AMIsMidnight(t *testing.T) { + midnight, err := Parse("3:04PM", "12:00AM") + if err != nil { + t.Fatal("error parsing date:", err) + } + if midnight.Hour != 0 { + t.Errorf("got %d; expect 0", midnight.Hour) + } + midnight, err = Parse("03:04PM", "12:00AM") + if err != nil { + t.Fatal("error parsing date:", err) + } + if midnight.Hour != 0 { + t.Errorf("got %d; expect 0", midnight.Hour) + } +} + // Check that a time without a Zone still produces a (numeric) time zone // when formatted with MST as a requested zone. func TestMissingZone(t *testing.T) { diff --git a/src/pkg/time/zoneinfo_windows.go b/src/pkg/time/zoneinfo_windows.go index c357eec62..83afdfb02 100644 --- a/src/pkg/time/zoneinfo_windows.go +++ b/src/pkg/time/zoneinfo_windows.go @@ -14,7 +14,7 @@ import ( // this year's rules for daylight savings time apply to all previous // and future years as well. -// TODO(brainman): use GetDynamicTimeZoneInformation, whenever posible (Vista and up), +// TODO(brainman): use GetDynamicTimeZoneInformation, whenever possible (Vista and up), // to improve on situation described in the bug above. type zone struct { @@ -46,7 +46,7 @@ func (z *zone) populate(bias, biasdelta int32, d *syscall.Systemtime, name []uin return } -// Pre-calculte cutoff time in seconds since the Unix epoch, if data is supplied in "absolute" format. +// Pre-calculate cutoff time in seconds since the Unix epoch, if data is supplied in "absolute" format. func (z *zone) preCalculateAbsSec() { if z.year != 0 { z.abssec = (&Time{z.year, int(z.month), int(z.day), int(z.hour), int(z.minute), int(z.second), 0, 0, ""}).Seconds() @@ -55,7 +55,7 @@ func (z *zone) preCalculateAbsSec() { } } -// Convert zone cutoff time to sec in number of seconds since the Unix epoch, given particualar year. +// Convert zone cutoff time to sec in number of seconds since the Unix epoch, given particular year. func (z *zone) cutoffSeconds(year int64) int64 { // Windows specifies daylight savings information in "day in month" format: // z.month is month number (1-12) @@ -96,7 +96,7 @@ func (z *zone) isBeforeCutoff(t *Time) bool { } type zoneinfo struct { - disabled bool // daylight saving time is not used localy + disabled bool // daylight saving time is not used locally offsetIfDisabled int januaryIsStd bool // is january 1 standard time? std, dst zone diff --git a/src/pkg/unicode/Makefile b/src/pkg/unicode/Makefile index 53f7229e7..55ed5b2d9 100644 --- a/src/pkg/unicode/Makefile +++ b/src/pkg/unicode/Makefile @@ -8,6 +8,7 @@ TARG=unicode GOFILES=\ casetables.go\ digit.go\ + graphic.go\ letter.go\ tables.go\ @@ -26,6 +27,7 @@ tables: maketables # Build (but do not run) maketables during testing, # just to make sure it still compiles. test: maketables +testshort: maketables # Downloads from www.unicode.org, so not part # of standard test scripts. diff --git a/src/pkg/unicode/digit.go b/src/pkg/unicode/digit.go index 471c4dfdc..6793fd7e5 100644 --- a/src/pkg/unicode/digit.go +++ b/src/pkg/unicode/digit.go @@ -6,7 +6,7 @@ package unicode // IsDigit reports whether the rune is a decimal digit. func IsDigit(rune int) bool { - if rune < 0x100 { // quick ASCII (Latin-1, really) check + if rune <= MaxLatin1 { return '0' <= rune && rune <= '9' } return Is(Digit, rune) diff --git a/src/pkg/unicode/digit_test.go b/src/pkg/unicode/digit_test.go index 9bbccde92..ae3c0ece9 100644 --- a/src/pkg/unicode/digit_test.go +++ b/src/pkg/unicode/digit_test.go @@ -118,7 +118,7 @@ func TestDigit(t *testing.T) { // Test that the special case in IsDigit agrees with the table func TestDigitOptimization(t *testing.T) { - for i := 0; i < 0x100; i++ { + for i := 0; i <= MaxLatin1; i++ { if Is(Digit, i) != IsDigit(i) { t.Errorf("IsDigit(U+%04X) disagrees with Is(Digit)", i) } diff --git a/src/pkg/unicode/graphic.go b/src/pkg/unicode/graphic.go new file mode 100644 index 000000000..d482aace2 --- /dev/null +++ b/src/pkg/unicode/graphic.go @@ -0,0 +1,132 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unicode + +// Bit masks for each code point under U+0100, for fast lookup. +const ( + pC = 1 << iota // a control character. + pP // a punctuation character. + pN // a numeral. + pS // a symbolic character. + pZ // a spacing character. + pLu // an upper-case letter. + pLl // a lower-case letter. + pp // a printable character according to Go's definition. + pg = pp | pZ // a graphical character according to the Unicode definition. +) + +// GraphicRanges defines the set of graphic characters according to Unicode. +var GraphicRanges = []*RangeTable{ + L, M, N, P, S, Zs, +} + +// PrintRanges defines the set of printable characters according to Go. +// ASCII space, U+0020, is handled separately. +var PrintRanges = []*RangeTable{ + L, M, N, P, S, +} + +// IsGraphic reports whether the rune is defined as a Graphic by Unicode. +// Such characters include letters, marks, numbers, punctuation, symbols, and +// spaces, from categories L, M, N, P, S, Zs. +func IsGraphic(rune int) bool { + // We cast to uint32 to avoid the extra test for negative, + // and in the index we cast to uint8 to avoid the range check. + if uint32(rune) <= MaxLatin1 { + return properties[uint8(rune)]&pg != 0 + } + return IsOneOf(GraphicRanges, rune) +} + +// IsPrint reports whether the rune is defined as printable by Go. Such +// characters include letters, marks, numbers, punctuation, symbols, and the +// ASCII space character, from categories L, M, N, P, S and the ASCII space +// character. This categorization is the same as IsGraphic except that the +// only spacing character is ASCII space, U+0020. +func IsPrint(rune int) bool { + if uint32(rune) <= MaxLatin1 { + return properties[uint8(rune)]&pp != 0 + } + return IsOneOf(PrintRanges, rune) +} + +// IsOneOf reports whether the rune is a member of one of the ranges. +// The rune is known to be above Latin-1. +func IsOneOf(set []*RangeTable, rune int) bool { + for _, inside := range set { + if Is(inside, rune) { + return true + } + } + return false +} + +// IsControl reports whether the rune is a control character. +// The C (Other) Unicode category includes more code points +// such as surrogates; use Is(C, rune) to test for them. +func IsControl(rune int) bool { + if uint32(rune) <= MaxLatin1 { + return properties[uint8(rune)]&pC != 0 + } + // All control characters are < Latin1Max. + return false +} + +// IsLetter reports whether the rune is a letter (category L). +func IsLetter(rune int) bool { + if uint32(rune) <= MaxLatin1 { + return properties[uint8(rune)]&(pLu|pLl) != 0 + } + return Is(Letter, rune) +} + +// IsMark reports whether the rune is a mark character (category M). +func IsMark(rune int) bool { + // There are no mark characters in Latin-1. + return Is(Mark, rune) +} + +// IsNumber reports whether the rune is a number (category N). +func IsNumber(rune int) bool { + if uint32(rune) <= MaxLatin1 { + return properties[uint8(rune)]&pN != 0 + } + return Is(Number, rune) +} + +// IsPunct reports whether the rune is a Unicode punctuation character +// (category P). +func IsPunct(rune int) bool { + if uint32(rune) <= MaxLatin1 { + return properties[uint8(rune)]&pP != 0 + } + return Is(Punct, rune) +} + +// IsSpace reports whether the rune is a space character as defined +// by Unicode's White Space property; in the Latin-1 space +// this is +// '\t', '\n', '\v', '\f', '\r', ' ', U+0085 (NEL), U+00A0 (NBSP). +// Other definitions of spacing characters are set by category +// Z and property Pattern_White_Space. +func IsSpace(rune int) bool { + // This property isn't the same as Z; special-case it. + if uint32(rune) <= MaxLatin1 { + switch rune { + case '\t', '\n', '\v', '\f', '\r', ' ', 0x85, 0xA0: + return true + } + return false + } + return Is(White_Space, rune) +} + +// IsSymbol reports whether the rune is a symbolic character. +func IsSymbol(rune int) bool { + if uint32(rune) <= MaxLatin1 { + return properties[uint8(rune)]&pS != 0 + } + return Is(Symbol, rune) +} diff --git a/src/pkg/unicode/graphic_test.go b/src/pkg/unicode/graphic_test.go new file mode 100644 index 000000000..77c679f7c --- /dev/null +++ b/src/pkg/unicode/graphic_test.go @@ -0,0 +1,122 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unicode_test + +import ( + "testing" + . "unicode" +) + +// Independently check that the special "Is" functions work +// in the Latin-1 range through the property table. + +func TestIsControlLatin1(t *testing.T) { + for i := 0; i <= MaxLatin1; i++ { + got := IsControl(i) + want := false + switch { + case 0x00 <= i && i <= 0x1F: + want = true + case 0x7F <= i && i <= 0x9F: + want = true + } + if got != want { + t.Errorf("%U incorrect: got %t; want %t", i, got, want) + } + } +} + +func TestIsLetterLatin1(t *testing.T) { + for i := 0; i <= MaxLatin1; i++ { + got := IsLetter(i) + want := Is(Letter, i) + if got != want { + t.Errorf("%U incorrect: got %t; want %t", i, got, want) + } + } +} + +func TestIsUpperLatin1(t *testing.T) { + for i := 0; i <= MaxLatin1; i++ { + got := IsUpper(i) + want := Is(Upper, i) + if got != want { + t.Errorf("%U incorrect: got %t; want %t", i, got, want) + } + } +} + +func TestIsLowerLatin1(t *testing.T) { + for i := 0; i <= MaxLatin1; i++ { + got := IsLower(i) + want := Is(Lower, i) + if got != want { + t.Errorf("%U incorrect: got %t; want %t", i, got, want) + } + } +} + +func TestNumberLatin1(t *testing.T) { + for i := 0; i <= MaxLatin1; i++ { + got := IsNumber(i) + want := Is(Number, i) + if got != want { + t.Errorf("%U incorrect: got %t; want %t", i, got, want) + } + } +} + +func TestIsPrintLatin1(t *testing.T) { + for i := 0; i <= MaxLatin1; i++ { + got := IsPrint(i) + want := IsOneOf(PrintRanges, i) + if i == ' ' { + want = true + } + if got != want { + t.Errorf("%U incorrect: got %t; want %t", i, got, want) + } + } +} + +func TestIsGraphicLatin1(t *testing.T) { + for i := 0; i <= MaxLatin1; i++ { + got := IsGraphic(i) + want := IsOneOf(GraphicRanges, i) + if got != want { + t.Errorf("%U incorrect: got %t; want %t", i, got, want) + } + } +} + +func TestIsPunctLatin1(t *testing.T) { + for i := 0; i <= MaxLatin1; i++ { + got := IsPunct(i) + want := Is(Punct, i) + if got != want { + t.Errorf("%U incorrect: got %t; want %t", i, got, want) + } + } +} + +func TestIsSpaceLatin1(t *testing.T) { + for i := 0; i <= MaxLatin1; i++ { + got := IsSpace(i) + want := Is(White_Space, i) + if got != want { + t.Errorf("%U incorrect: got %t; want %t", i, got, want) + } + } +} + +func TestIsSymbolLatin1(t *testing.T) { + for i := 0; i <= MaxLatin1; i++ { + got := IsSymbol(i) + want := Is(Symbol, i) + if got != want { + t.Errorf("%U incorrect: got %t; want %t", i, got, want) + } + } +} diff --git a/src/pkg/unicode/letter.go b/src/pkg/unicode/letter.go index 382c6eb3f..a0c55bbf7 100644 --- a/src/pkg/unicode/letter.go +++ b/src/pkg/unicode/letter.go @@ -9,15 +9,35 @@ package unicode const ( MaxRune = 0x10FFFF // Maximum valid Unicode code point. ReplacementChar = 0xFFFD // Represents invalid code points. + MaxASCII = 0x7F // maximum ASCII value. + MaxLatin1 = 0xFF // maximum Latin-1 value. ) +// RangeTable defines a set of Unicode code points by listing the ranges of +// code points within the set. The ranges are listed in two slices +// to save space: a slice of 16-bit ranges and a slice of 32-bit ranges. +// The two slices must be in sorted order and non-overlapping. +// Also, R32 should contain only values >= 0x10000 (1<<16). +type RangeTable struct { + R16 []Range16 + R32 []Range32 +} -// The representation of a range of Unicode code points. The range runs from Lo to Hi +// Range16 represents of a range of 16-bit Unicode code points. The range runs from Lo to Hi // inclusive and has the specified stride. -type Range struct { - Lo int - Hi int - Stride int +type Range16 struct { + Lo uint16 + Hi uint16 + Stride uint16 +} + +// Range32 represents of a range of Unicode code points and is used when one or +// more of the values will not fit in 16 bits. The range runs from Lo to Hi +// inclusive and has the specified stride. Lo and Hi must always be >= 1<<16. +type Range32 struct { + Lo uint32 + Hi uint32 + Stride uint32 } // CaseRange represents a range of Unicode code points for simple (one @@ -31,8 +51,8 @@ type Range struct { // {UpperLower, UpperLower, UpperLower} // The constant UpperLower has an otherwise impossible delta value. type CaseRange struct { - Lo int - Hi int + Lo uint32 + Hi uint32 Delta d } @@ -60,22 +80,28 @@ const ( UpperLower = MaxRune + 1 // (Cannot be a valid delta.) ) -// Is tests whether rune is in the specified table of ranges. -func Is(ranges []Range, rune int) bool { - // common case: rune is ASCII or Latin-1 - if rune < 0x100 { - for _, r := range ranges { - if rune > r.Hi { - continue - } - if rune < r.Lo { - return false - } +// is16 uses binary search to test whether rune is in the specified slice of 16-bit ranges. +func is16(ranges []Range16, rune uint16) bool { + // binary search over ranges + lo := 0 + hi := len(ranges) + for lo < hi { + m := lo + (hi-lo)/2 + r := ranges[m] + if r.Lo <= rune && rune <= r.Hi { return (rune-r.Lo)%r.Stride == 0 } - return false + if rune < r.Lo { + hi = m + } else { + lo = m + 1 + } } + return false +} +// is32 uses binary search to test whether rune is in the specified slice of 32-bit ranges. +func is32(ranges []Range32, rune uint32) bool { // binary search over ranges lo := 0 hi := len(ranges) @@ -94,51 +120,60 @@ func Is(ranges []Range, rune int) bool { return false } +// Is tests whether rune is in the specified table of ranges. +func Is(rangeTab *RangeTable, rune int) bool { + // common case: rune is ASCII or Latin-1. + if uint32(rune) <= MaxLatin1 { + // Only need to check R16, since R32 is always >= 1<<16. + r16 := uint16(rune) + for _, r := range rangeTab.R16 { + if r16 > r.Hi { + continue + } + if r16 < r.Lo { + return false + } + return (r16-r.Lo)%r.Stride == 0 + } + return false + } + r16 := rangeTab.R16 + if len(r16) > 0 && rune <= int(r16[len(r16)-1].Hi) { + return is16(r16, uint16(rune)) + } + r32 := rangeTab.R32 + if len(r32) > 0 && rune >= int(r32[0].Lo) { + return is32(r32, uint32(rune)) + } + return false +} + // IsUpper reports whether the rune is an upper case letter. func IsUpper(rune int) bool { - if rune < 0x80 { // quick ASCII check - return 'A' <= rune && rune <= 'Z' + // See comment in IsGraphic. + if uint32(rune) <= MaxLatin1 { + return properties[uint8(rune)]&pLu != 0 } return Is(Upper, rune) } // IsLower reports whether the rune is a lower case letter. func IsLower(rune int) bool { - if rune < 0x80 { // quick ASCII check - return 'a' <= rune && rune <= 'z' + // See comment in IsGraphic. + if uint32(rune) <= MaxLatin1 { + return properties[uint8(rune)]&pLl != 0 } return Is(Lower, rune) } // IsTitle reports whether the rune is a title case letter. func IsTitle(rune int) bool { - if rune < 0x80 { // quick ASCII check + if rune <= MaxLatin1 { return false } return Is(Title, rune) } -// IsLetter reports whether the rune is a letter. -func IsLetter(rune int) bool { - if rune < 0x80 { // quick ASCII check - rune &^= 'a' - 'A' - return 'A' <= rune && rune <= 'Z' - } - return Is(Letter, rune) -} - -// IsSpace reports whether the rune is a white space character. -func IsSpace(rune int) bool { - if rune <= 0xFF { // quick Latin-1 check - switch rune { - case '\t', '\n', '\v', '\f', '\r', ' ', 0x85, 0xA0: - return true - } - return false - } - return Is(White_Space, rune) -} - // to maps the rune using the specified case mapping. func to(_case int, rune int, caseRange []CaseRange) int { if _case < 0 || MaxCase <= _case { @@ -150,7 +185,7 @@ func to(_case int, rune int, caseRange []CaseRange) int { for lo < hi { m := lo + (hi-lo)/2 r := caseRange[m] - if r.Lo <= rune && rune <= r.Hi { + if int(r.Lo) <= rune && rune <= int(r.Hi) { delta := int(r.Delta[_case]) if delta > MaxRune { // In an Upper-Lower sequence, which always starts with @@ -163,11 +198,11 @@ func to(_case int, rune int, caseRange []CaseRange) int { // bit in the sequence offset. // The constants UpperCase and TitleCase are even while LowerCase // is odd so we take the low bit from _case. - return r.Lo + ((rune-r.Lo)&^1 | _case&1) + return int(r.Lo) + ((rune-int(r.Lo))&^1 | _case&1) } return rune + delta } - if rune < r.Lo { + if rune < int(r.Lo) { hi = m } else { lo = m + 1 @@ -183,7 +218,7 @@ func To(_case int, rune int) int { // ToUpper maps the rune to upper case. func ToUpper(rune int) int { - if rune < 0x80 { // quick ASCII check + if rune <= MaxASCII { if 'a' <= rune && rune <= 'z' { rune -= 'a' - 'A' } @@ -194,7 +229,7 @@ func ToUpper(rune int) int { // ToLower maps the rune to lower case. func ToLower(rune int) int { - if rune < 0x80 { // quick ASCII check + if rune <= MaxASCII { if 'A' <= rune && rune <= 'Z' { rune += 'a' - 'A' } @@ -205,7 +240,7 @@ func ToLower(rune int) int { // ToTitle maps the rune to title case. func ToTitle(rune int) int { - if rune < 0x80 { // quick ASCII check + if rune <= MaxASCII { if 'a' <= rune && rune <= 'z' { // title case is upper case for ASCII rune -= 'a' - 'A' } diff --git a/src/pkg/unicode/letter_test.go b/src/pkg/unicode/letter_test.go index 432ffb671..4c24ffc51 100644 --- a/src/pkg/unicode/letter_test.go +++ b/src/pkg/unicode/letter_test.go @@ -323,7 +323,7 @@ func TestIsSpace(t *testing.T) { // Check that the optimizations for IsLetter etc. agree with the tables. // We only need to check the Latin-1 range. func TestLetterOptimizations(t *testing.T) { - for i := 0; i < 0x100; i++ { + for i := 0; i <= MaxLatin1; i++ { if Is(Letter, i) != IsLetter(i) { t.Errorf("IsLetter(U+%04X) disagrees with Is(Letter)", i) } diff --git a/src/pkg/unicode/maketables.go b/src/pkg/unicode/maketables.go index 0c367673e..655fe46e4 100644 --- a/src/pkg/unicode/maketables.go +++ b/src/pkg/unicode/maketables.go @@ -28,6 +28,8 @@ func main() { printScriptOrProperty(false) printScriptOrProperty(true) printCases() + printLatinProperties() + printSizes() } var dataURL = flag.String("data", "", "full URL for UnicodeData.txt; defaults to --url/UnicodeData.txt") @@ -53,7 +55,17 @@ var test = flag.Bool("test", var scriptRe = regexp.MustCompile(`^([0-9A-F]+)(\.\.[0-9A-F]+)? *; ([A-Za-z_]+)$`) var logger = log.New(os.Stderr, "", log.Lshortfile) -var category = map[string]bool{"letter": true} // Nd Lu etc. letter is a special case +var category = map[string]bool{ + // Nd Lu etc. + // We use one-character names to identify merged categories + "L": true, // Lu Ll Lt Lm Lo + "P": true, // Pc Pd Ps Pe Pu Pf Po + "M": true, // Mn Mc Me + "N": true, // Nd Nl No + "S": true, // Sm Sc Sk So + "Z": true, // Zs Zl Zp + "C": true, // Cc Cf Cs Co Cn +} // UnicodeData.txt has form: // 0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;; @@ -246,19 +258,16 @@ func version() string { return "Unknown" } -func letterOp(code int) bool { - switch chars[code].category { - case "Lu", "Ll", "Lt", "Lm", "Lo": - return true - } - return false +func categoryOp(code int, class uint8) bool { + category := chars[code].category + return len(category) > 0 && category[0] == class } func loadChars() { if *dataURL == "" { flag.Set("data", *url+"UnicodeData.txt") } - resp, _, err := http.Get(*dataURL) + resp, err := http.Get(*dataURL) if err != nil { logger.Fatal(err) } @@ -278,16 +287,16 @@ func loadChars() { switch parseCategory(line[0 : len(line)-1]) { case SNormal: if first != 0 { - logger.Fatalf("bad state normal at U+%04X", lastChar) + logger.Fatalf("bad state normal at %U", lastChar) } case SFirst: if first != 0 { - logger.Fatalf("bad state first at U+%04X", lastChar) + logger.Fatalf("bad state first at %U", lastChar) } first = lastChar case SLast: if first == 0 { - logger.Fatalf("bad state last at U+%04X", lastChar) + logger.Fatalf("bad state last at %U", lastChar) } for i := first + 1; i <= lastChar; i++ { chars[i] = chars[first] @@ -299,6 +308,15 @@ func loadChars() { resp.Body.Close() } +const progHeader = `// Generated by running +// maketables --tables=%s --data=%s +// DO NOT EDIT + +package unicode + +` + + func printCategories() { if *tablelist == "" { return @@ -312,20 +330,14 @@ func printCategories() { fullCategoryTest(list) return } - fmt.Printf( - "// Generated by running\n"+ - "// maketables --tables=%s --data=%s\n"+ - "// DO NOT EDIT\n\n"+ - "package unicode\n\n", - *tablelist, - *dataURL) + fmt.Printf(progHeader, *tablelist, *dataURL) fmt.Println("// Version is the Unicode edition from which the tables are derived.") fmt.Printf("const Version = %q\n\n", version()) if *tablelist == "all" { fmt.Println("// Categories is the set of Unicode data tables.") - fmt.Println("var Categories = map[string] []Range {") + fmt.Println("var Categories = map[string] *RangeTable {") for k := range category { fmt.Printf("\t%q: %s,\n", k, k) } @@ -344,8 +356,27 @@ func printCategories() { // Cases deserving special comments varDecl := "" switch name { - case "letter": - varDecl = "\tLetter = letter; // Letter is the set of Unicode letters.\n" + case "C": + varDecl = "\tOther = _C; // Other/C is the set of Unicode control and special characters, category C.\n" + varDecl += "\tC = _C\n" + case "L": + varDecl = "\tLetter = _L; // Letter/L is the set of Unicode letters, category L.\n" + varDecl += "\tL = _L\n" + case "M": + varDecl = "\tMark = _M; // Mark/M is the set of Unicode mark characters, category M.\n" + varDecl += "\tM = _M\n" + case "N": + varDecl = "\tNumber = _N; // Number/N is the set of Unicode number characters, category N.\n" + varDecl += "\tN = _N\n" + case "P": + varDecl = "\tPunct = _P; // Punct/P is the set of Unicode punctuation characters, category P.\n" + varDecl += "\tP = _P\n" + case "S": + varDecl = "\tSymbol = _S; // Symbol/S is the set of Unicode symbol characters, category S.\n" + varDecl += "\tS = _S\n" + case "Z": + varDecl = "\tSpace = _Z; // Space/Z is the set of Unicode space characters, category Z.\n" + varDecl += "\tZ = _Z\n" case "Nd": varDecl = "\tDigit = _Nd; // Digit is the set of Unicode characters with the \"decimal digit\" property.\n" case "Lu": @@ -355,21 +386,22 @@ func printCategories() { case "Lt": varDecl = "\tTitle = _Lt; // Title is the set of Unicode title case letters.\n" } - if name != "letter" { + if len(name) > 1 { varDecl += fmt.Sprintf( "\t%s = _%s; // %s is the set of Unicode characters in category %s.\n", name, name, name, name) } decl[ndecl] = varDecl ndecl++ - if name == "letter" { // special case + if len(name) == 1 { // unified categories + decl := fmt.Sprintf("var _%s = &RangeTable{\n", name) dumpRange( - "var letter = []Range {\n", - letterOp) + decl, + func(code int) bool { return categoryOp(code, name[0]) }) continue } dumpRange( - fmt.Sprintf("var _%s = []Range {\n", name), + fmt.Sprintf("var _%s = &RangeTable{\n", name), func(code int) bool { return chars[code].category == name }) } decl.Sort() @@ -382,12 +414,15 @@ func printCategories() { type Op func(code int) bool -const format = "\t{0x%04x, 0x%04x, %d},\n" +const format = "\t\t{0x%04x, 0x%04x, %d},\n" func dumpRange(header string, inCategory Op) { fmt.Print(header) next := 0 + fmt.Print("\tR16: []Range16{\n") // one Range for each iteration + count := &range16Count + size := 16 for { // look for start of range for next < len(chars) && !inCategory(next) { @@ -427,24 +462,49 @@ func dumpRange(header string, inCategory Op) { break } } - fmt.Printf(format, lo, hi, stride) + size, count = printRange(uint32(lo), uint32(hi), uint32(stride), size, count) // next range: start looking where this range ends next = hi + 1 } + fmt.Print("\t},\n") fmt.Print("}\n\n") } +func printRange(lo, hi, stride uint32, size int, count *int) (int, *int) { + if size == 16 && hi >= 1<<16 { + if lo < 1<<16 { + if lo+stride != hi { + logger.Fatalf("unexpected straddle: %U %U %d", lo, hi, stride) + } + // No range contains U+FFFF as an instance, so split + // the range into two entries. That way we can maintain + // the invariant that R32 contains only >= 1<<16. + fmt.Printf(format, lo, lo, 1) + lo = hi + stride = 1 + *count++ + } + fmt.Print("\t},\n") + fmt.Print("\tR32: []Range32{\n") + size = 32 + count = &range32Count + } + fmt.Printf(format, lo, hi, stride) + *count++ + return size, count +} + func fullCategoryTest(list []string) { for _, name := range list { if _, ok := category[name]; !ok { logger.Fatal("unknown category", name) } r, ok := unicode.Categories[name] - if !ok { - logger.Fatal("unknown table", name) + if !ok && len(name) > 1 { + logger.Fatalf("unknown table %q", name) } - if name == "letter" { - verifyRange(name, letterOp, r) + if len(name) == 1 { + verifyRange(name, func(code int) bool { return categoryOp(code, name[0]) }, r) } else { verifyRange( name, @@ -454,12 +514,17 @@ func fullCategoryTest(list []string) { } } -func verifyRange(name string, inCategory Op, table []unicode.Range) { +func verifyRange(name string, inCategory Op, table *unicode.RangeTable) { + count := 0 for i := range chars { web := inCategory(i) pkg := unicode.Is(table, i) if web != pkg { - fmt.Fprintf(os.Stderr, "%s: U+%04X: web=%t pkg=%t\n", name, i, web, pkg) + fmt.Fprintf(os.Stderr, "%s: %U: web=%t pkg=%t\n", name, i, web, pkg) + count++ + if count > 10 { + break + } } } } @@ -497,22 +562,22 @@ func parseScript(line string, scripts map[string][]Script) { } // The script tables have a lot of adjacent elements. Fold them together. -func foldAdjacent(r []Script) []unicode.Range { - s := make([]unicode.Range, 0, len(r)) +func foldAdjacent(r []Script) []unicode.Range32 { + s := make([]unicode.Range32, 0, len(r)) j := 0 for i := 0; i < len(r); i++ { - if j > 0 && int(r[i].lo) == s[j-1].Hi+1 { - s[j-1].Hi = int(r[i].hi) + if j > 0 && r[i].lo == s[j-1].Hi+1 { + s[j-1].Hi = r[i].hi } else { s = s[0 : j+1] - s[j] = unicode.Range{int(r[i].lo), int(r[i].hi), 1} + s[j] = unicode.Range32{uint32(r[i].lo), uint32(r[i].hi), 1} j++ } } return s } -func fullScriptTest(list []string, installed map[string][]unicode.Range, scripts map[string][]Script) { +func fullScriptTest(list []string, installed map[string]*unicode.RangeTable, scripts map[string][]Script) { for _, name := range list { if _, ok := scripts[name]; !ok { logger.Fatal("unknown script", name) @@ -524,7 +589,7 @@ func fullScriptTest(list []string, installed map[string][]unicode.Range, scripts for _, script := range scripts[name] { for r := script.lo; r <= script.hi; r++ { if !unicode.Is(installed[name], int(r)) { - fmt.Fprintf(os.Stderr, "U+%04X: not in script %s\n", r, name) + fmt.Fprintf(os.Stderr, "%U: not in script %s\n", r, name) } } } @@ -549,7 +614,7 @@ func printScriptOrProperty(doProps bool) { return } var err os.Error - resp, _, err := http.Get(*url + file) + resp, err := http.Get(*url + file) if err != nil { logger.Fatal(err) } @@ -589,10 +654,10 @@ func printScriptOrProperty(doProps bool) { if flaglist == "all" { if doProps { fmt.Println("// Properties is the set of Unicode property tables.") - fmt.Println("var Properties = map[string] []Range {") + fmt.Println("var Properties = map[string] *RangeTable{") } else { fmt.Println("// Scripts is the set of Unicode script tables.") - fmt.Println("var Scripts = map[string] []Range {") + fmt.Println("var Scripts = map[string] *RangeTable{") } for k := range table { fmt.Printf("\t%q: %s,\n", k, k) @@ -613,11 +678,15 @@ func printScriptOrProperty(doProps bool) { name, name, name, name) } ndecl++ - fmt.Printf("var _%s = []Range {\n", name) + fmt.Printf("var _%s = &RangeTable {\n", name) + fmt.Print("\tR16: []Range16{\n") ranges := foldAdjacent(table[name]) + size := 16 + count := &range16Count for _, s := range ranges { - fmt.Printf(format, s.Lo, s.Hi, s.Stride) + size, count = printRange(s.Lo, s.Hi, s.Stride, size, count) } + fmt.Print("\t},\n") fmt.Print("}\n\n") } decl.Sort() @@ -808,7 +877,7 @@ func printCaseRange(lo, hi *caseState) { fmt.Printf("\t{0x%04X, 0x%04X, d{UpperLower, UpperLower, UpperLower}},\n", lo.point, hi.point) case hi.point > lo.point && lo.isLowerUpper(): - logger.Fatalf("LowerUpper sequence: should not happen: U+%04X. If it's real, need to fix To()", lo.point) + logger.Fatalf("LowerUpper sequence: should not happen: %U. If it's real, need to fix To()", lo.point) fmt.Printf("\t{0x%04X, 0x%04X, d{LowerUpper, LowerUpper, LowerUpper}},\n", lo.point, hi.point) default: @@ -831,17 +900,67 @@ func fullCaseTest() { lower := unicode.ToLower(i) want := caseIt(i, c.lowerCase) if lower != want { - fmt.Fprintf(os.Stderr, "lower U+%04X should be U+%04X is U+%04X\n", i, want, lower) + fmt.Fprintf(os.Stderr, "lower %U should be %U is %U\n", i, want, lower) } upper := unicode.ToUpper(i) want = caseIt(i, c.upperCase) if upper != want { - fmt.Fprintf(os.Stderr, "upper U+%04X should be U+%04X is U+%04X\n", i, want, upper) + fmt.Fprintf(os.Stderr, "upper %U should be %U is %U\n", i, want, upper) } title := unicode.ToTitle(i) want = caseIt(i, c.titleCase) if title != want { - fmt.Fprintf(os.Stderr, "title U+%04X should be U+%04X is U+%04X\n", i, want, title) + fmt.Fprintf(os.Stderr, "title %U should be %U is %U\n", i, want, title) } } } + +func printLatinProperties() { + if *test { + return + } + fmt.Println("var properties = [MaxLatin1+1]uint8{") + for code := 0; code <= unicode.MaxLatin1; code++ { + var property string + switch chars[code].category { + case "Cc", "": // NUL has no category. + property = "pC" + case "Cf": // soft hyphen, unique category, not printable. + property = "0" + case "Ll": + property = "pLl | pp" + case "Lu": + property = "pLu | pp" + case "Nd", "No": + property = "pN | pp" + case "Pc", "Pd", "Pe", "Pf", "Pi", "Po", "Ps": + property = "pP | pp" + case "Sc", "Sk", "Sm", "So": + property = "pS | pp" + case "Zs": + property = "pZ" + default: + logger.Fatalf("%U has unknown category %q", code, chars[code].category) + } + // Special case + if code == ' ' { + property = "pZ | pp" + } + fmt.Printf("\t0x%.2X: %s, // %q\n", code, property, code) + } + fmt.Println("}") +} + +var range16Count = 0 // Number of entries in the 16-bit range tables. +var range32Count = 0 // Number of entries in the 32-bit range tables. + +func printSizes() { + if *test { + return + } + fmt.Println() + fmt.Printf("// Range entries: %d 16-bit, %d 32-bit, %d total.\n", range16Count, range32Count, range16Count+range32Count) + range16Bytes := range16Count * 3 * 2 + range32Bytes := range32Count * 3 * 4 + fmt.Printf("// Range bytes: %d 16-bit, %d 32-bit, %d total.\n", range16Bytes, range32Bytes, range16Bytes+range32Bytes) +} diff --git a/src/pkg/unicode/script_test.go b/src/pkg/unicode/script_test.go index ff452b75c..b37ad1836 100644 --- a/src/pkg/unicode/script_test.go +++ b/src/pkg/unicode/script_test.go @@ -149,7 +149,14 @@ var inCategoryTest = []T{ {0x2028, "Zl"}, {0x2029, "Zp"}, {0x202f, "Zs"}, - {0x04aa, "letter"}, + // Unifieds. + {0x04aa, "L"}, + {0x0009, "C"}, + {0x1712, "M"}, + {0x0031, "N"}, + {0x00bb, "P"}, + {0x00a2, "S"}, + {0x00a0, "Z"}, } var inPropTest = []T{ @@ -197,13 +204,13 @@ func TestScripts(t *testing.T) { t.Fatal(test.script, "not a known script") } if !Is(Scripts[test.script], test.rune) { - t.Errorf("IsScript(%#x, %s) = false, want true", test.rune, test.script) + t.Errorf("IsScript(%U, %s) = false, want true", test.rune, test.script) } notTested[test.script] = false, false } for _, test := range outTest { if Is(Scripts[test.script], test.rune) { - t.Errorf("IsScript(%#x, %s) = true, want false", test.rune, test.script) + t.Errorf("IsScript(%U, %s) = true, want false", test.rune, test.script) } } for k := range notTested { @@ -221,7 +228,7 @@ func TestCategories(t *testing.T) { t.Fatal(test.script, "not a known category") } if !Is(Categories[test.script], test.rune) { - t.Errorf("IsCategory(%#x, %s) = false, want true", test.rune, test.script) + t.Errorf("IsCategory(%U, %s) = false, want true", test.rune, test.script) } notTested[test.script] = false, false } @@ -240,7 +247,7 @@ func TestProperties(t *testing.T) { t.Fatal(test.script, "not a known prop") } if !Is(Properties[test.script], test.rune) { - t.Errorf("IsCategory(%#x, %s) = false, want true", test.rune, test.script) + t.Errorf("IsCategory(%U, %s) = false, want true", test.rune, test.script) } notTested[test.script] = false, false } diff --git a/src/pkg/unicode/tables.go b/src/pkg/unicode/tables.go index 89fd99411..32681a8c0 100644 --- a/src/pkg/unicode/tables.go +++ b/src/pkg/unicode/tables.go @@ -8,2009 +8,2747 @@ package unicode const Version = "6.0.0" // Categories is the set of Unicode data tables. -var Categories = map[string][]Range{ - "Lm": Lm, - "Ll": Ll, - "Me": Me, - "Mc": Mc, - "Mn": Mn, - "Zl": Zl, - "letter": letter, - "Zp": Zp, - "Zs": Zs, - "Cs": Cs, - "Co": Co, - "Cf": Cf, - "Cc": Cc, - "Po": Po, - "Pi": Pi, - "Pf": Pf, - "Pe": Pe, - "Pd": Pd, - "Pc": Pc, - "Ps": Ps, - "Nd": Nd, - "Nl": Nl, - "No": No, - "So": So, - "Sm": Sm, - "Sk": Sk, - "Sc": Sc, - "Lu": Lu, - "Lt": Lt, - "Lo": Lo, -} - -var _Lm = []Range{ - {0x02b0, 0x02c1, 1}, - {0x02c6, 0x02d1, 1}, - {0x02e0, 0x02e4, 1}, - {0x02ec, 0x02ee, 2}, - {0x0374, 0x037a, 6}, - {0x0559, 0x0640, 231}, - {0x06e5, 0x06e6, 1}, - {0x07f4, 0x07f5, 1}, - {0x07fa, 0x081a, 32}, - {0x0824, 0x0828, 4}, - {0x0971, 0x0e46, 1237}, - {0x0ec6, 0x10fc, 566}, - {0x17d7, 0x1843, 108}, - {0x1aa7, 0x1c78, 465}, - {0x1c79, 0x1c7d, 1}, - {0x1d2c, 0x1d61, 1}, - {0x1d78, 0x1d9b, 35}, - {0x1d9c, 0x1dbf, 1}, - {0x2071, 0x207f, 14}, - {0x2090, 0x209c, 1}, - {0x2c7d, 0x2d6f, 242}, - {0x2e2f, 0x3005, 470}, - {0x3031, 0x3035, 1}, - {0x303b, 0x309d, 98}, - {0x309e, 0x30fc, 94}, - {0x30fd, 0x30fe, 1}, - {0xa015, 0xa4f8, 1251}, - {0xa4f9, 0xa4fd, 1}, - {0xa60c, 0xa67f, 115}, - {0xa717, 0xa71f, 1}, - {0xa770, 0xa788, 24}, - {0xa9cf, 0xaa70, 161}, - {0xaadd, 0xff70, 21651}, - {0xff9e, 0xff9f, 1}, -} - -var _Ll = []Range{ - {0x0061, 0x007a, 1}, - {0x00aa, 0x00b5, 11}, - {0x00ba, 0x00df, 37}, - {0x00e0, 0x00f6, 1}, - {0x00f8, 0x00ff, 1}, - {0x0101, 0x0137, 2}, - {0x0138, 0x0148, 2}, - {0x0149, 0x0177, 2}, - {0x017a, 0x017e, 2}, - {0x017f, 0x0180, 1}, - {0x0183, 0x0185, 2}, - {0x0188, 0x018c, 4}, - {0x018d, 0x0192, 5}, - {0x0195, 0x0199, 4}, - {0x019a, 0x019b, 1}, - {0x019e, 0x01a1, 3}, - {0x01a3, 0x01a5, 2}, - {0x01a8, 0x01aa, 2}, - {0x01ab, 0x01ad, 2}, - {0x01b0, 0x01b4, 4}, - {0x01b6, 0x01b9, 3}, - {0x01ba, 0x01bd, 3}, - {0x01be, 0x01bf, 1}, - {0x01c6, 0x01cc, 3}, - {0x01ce, 0x01dc, 2}, - {0x01dd, 0x01ef, 2}, - {0x01f0, 0x01f3, 3}, - {0x01f5, 0x01f9, 4}, - {0x01fb, 0x0233, 2}, - {0x0234, 0x0239, 1}, - {0x023c, 0x023f, 3}, - {0x0240, 0x0242, 2}, - {0x0247, 0x024f, 2}, - {0x0250, 0x0293, 1}, - {0x0295, 0x02af, 1}, - {0x0371, 0x0373, 2}, - {0x0377, 0x037b, 4}, - {0x037c, 0x037d, 1}, - {0x0390, 0x03ac, 28}, - {0x03ad, 0x03ce, 1}, - {0x03d0, 0x03d1, 1}, - {0x03d5, 0x03d7, 1}, - {0x03d9, 0x03ef, 2}, - {0x03f0, 0x03f3, 1}, - {0x03f5, 0x03fb, 3}, - {0x03fc, 0x0430, 52}, - {0x0431, 0x045f, 1}, - {0x0461, 0x0481, 2}, - {0x048b, 0x04bf, 2}, - {0x04c2, 0x04ce, 2}, - {0x04cf, 0x0527, 2}, - {0x0561, 0x0587, 1}, - {0x1d00, 0x1d2b, 1}, - {0x1d62, 0x1d77, 1}, - {0x1d79, 0x1d9a, 1}, - {0x1e01, 0x1e95, 2}, - {0x1e96, 0x1e9d, 1}, - {0x1e9f, 0x1eff, 2}, - {0x1f00, 0x1f07, 1}, - {0x1f10, 0x1f15, 1}, - {0x1f20, 0x1f27, 1}, - {0x1f30, 0x1f37, 1}, - {0x1f40, 0x1f45, 1}, - {0x1f50, 0x1f57, 1}, - {0x1f60, 0x1f67, 1}, - {0x1f70, 0x1f7d, 1}, - {0x1f80, 0x1f87, 1}, - {0x1f90, 0x1f97, 1}, - {0x1fa0, 0x1fa7, 1}, - {0x1fb0, 0x1fb4, 1}, - {0x1fb6, 0x1fb7, 1}, - {0x1fbe, 0x1fc2, 4}, - {0x1fc3, 0x1fc4, 1}, - {0x1fc6, 0x1fc7, 1}, - {0x1fd0, 0x1fd3, 1}, - {0x1fd6, 0x1fd7, 1}, - {0x1fe0, 0x1fe7, 1}, - {0x1ff2, 0x1ff4, 1}, - {0x1ff6, 0x1ff7, 1}, - {0x210a, 0x210e, 4}, - {0x210f, 0x2113, 4}, - {0x212f, 0x2139, 5}, - {0x213c, 0x213d, 1}, - {0x2146, 0x2149, 1}, - {0x214e, 0x2184, 54}, - {0x2c30, 0x2c5e, 1}, - {0x2c61, 0x2c65, 4}, - {0x2c66, 0x2c6c, 2}, - {0x2c71, 0x2c73, 2}, - {0x2c74, 0x2c76, 2}, - {0x2c77, 0x2c7c, 1}, - {0x2c81, 0x2ce3, 2}, - {0x2ce4, 0x2cec, 8}, - {0x2cee, 0x2d00, 18}, - {0x2d01, 0x2d25, 1}, - {0xa641, 0xa66d, 2}, - {0xa681, 0xa697, 2}, - {0xa723, 0xa72f, 2}, - {0xa730, 0xa731, 1}, - {0xa733, 0xa771, 2}, - {0xa772, 0xa778, 1}, - {0xa77a, 0xa77c, 2}, - {0xa77f, 0xa787, 2}, - {0xa78c, 0xa78e, 2}, - {0xa791, 0xa7a1, 16}, - {0xa7a3, 0xa7a9, 2}, - {0xa7fa, 0xfb00, 21254}, - {0xfb01, 0xfb06, 1}, - {0xfb13, 0xfb17, 1}, - {0xff41, 0xff5a, 1}, - {0x10428, 0x1044f, 1}, - {0x1d41a, 0x1d433, 1}, - {0x1d44e, 0x1d454, 1}, - {0x1d456, 0x1d467, 1}, - {0x1d482, 0x1d49b, 1}, - {0x1d4b6, 0x1d4b9, 1}, - {0x1d4bb, 0x1d4bd, 2}, - {0x1d4be, 0x1d4c3, 1}, - {0x1d4c5, 0x1d4cf, 1}, - {0x1d4ea, 0x1d503, 1}, - {0x1d51e, 0x1d537, 1}, - {0x1d552, 0x1d56b, 1}, - {0x1d586, 0x1d59f, 1}, - {0x1d5ba, 0x1d5d3, 1}, - {0x1d5ee, 0x1d607, 1}, - {0x1d622, 0x1d63b, 1}, - {0x1d656, 0x1d66f, 1}, - {0x1d68a, 0x1d6a5, 1}, - {0x1d6c2, 0x1d6da, 1}, - {0x1d6dc, 0x1d6e1, 1}, - {0x1d6fc, 0x1d714, 1}, - {0x1d716, 0x1d71b, 1}, - {0x1d736, 0x1d74e, 1}, - {0x1d750, 0x1d755, 1}, - {0x1d770, 0x1d788, 1}, - {0x1d78a, 0x1d78f, 1}, - {0x1d7aa, 0x1d7c2, 1}, - {0x1d7c4, 0x1d7c9, 1}, - {0x1d7cb, 0x1d7cb, 1}, -} - -var _Me = []Range{ - {0x0488, 0x0489, 1}, - {0x20dd, 0x20e0, 1}, - {0x20e2, 0x20e4, 1}, - {0xa670, 0xa672, 1}, -} - -var _Mc = []Range{ - {0x0903, 0x093b, 56}, - {0x093e, 0x0940, 1}, - {0x0949, 0x094c, 1}, - {0x094e, 0x094f, 1}, - {0x0982, 0x0983, 1}, - {0x09be, 0x09c0, 1}, - {0x09c7, 0x09c8, 1}, - {0x09cb, 0x09cc, 1}, - {0x09d7, 0x0a03, 44}, - {0x0a3e, 0x0a40, 1}, - {0x0a83, 0x0abe, 59}, - {0x0abf, 0x0ac0, 1}, - {0x0ac9, 0x0acb, 2}, - {0x0acc, 0x0b02, 54}, - {0x0b03, 0x0b3e, 59}, - {0x0b40, 0x0b47, 7}, - {0x0b48, 0x0b4b, 3}, - {0x0b4c, 0x0b57, 11}, - {0x0bbe, 0x0bbf, 1}, - {0x0bc1, 0x0bc2, 1}, - {0x0bc6, 0x0bc8, 1}, - {0x0bca, 0x0bcc, 1}, - {0x0bd7, 0x0c01, 42}, - {0x0c02, 0x0c03, 1}, - {0x0c41, 0x0c44, 1}, - {0x0c82, 0x0c83, 1}, - {0x0cbe, 0x0cc0, 2}, - {0x0cc1, 0x0cc4, 1}, - {0x0cc7, 0x0cc8, 1}, - {0x0cca, 0x0ccb, 1}, - {0x0cd5, 0x0cd6, 1}, - {0x0d02, 0x0d03, 1}, - {0x0d3e, 0x0d40, 1}, - {0x0d46, 0x0d48, 1}, - {0x0d4a, 0x0d4c, 1}, - {0x0d57, 0x0d82, 43}, - {0x0d83, 0x0dcf, 76}, - {0x0dd0, 0x0dd1, 1}, - {0x0dd8, 0x0ddf, 1}, - {0x0df2, 0x0df3, 1}, - {0x0f3e, 0x0f3f, 1}, - {0x0f7f, 0x102b, 172}, - {0x102c, 0x1031, 5}, - {0x1038, 0x103b, 3}, - {0x103c, 0x1056, 26}, - {0x1057, 0x1062, 11}, - {0x1063, 0x1064, 1}, - {0x1067, 0x106d, 1}, - {0x1083, 0x1084, 1}, - {0x1087, 0x108c, 1}, - {0x108f, 0x109a, 11}, - {0x109b, 0x109c, 1}, - {0x17b6, 0x17be, 8}, - {0x17bf, 0x17c5, 1}, - {0x17c7, 0x17c8, 1}, - {0x1923, 0x1926, 1}, - {0x1929, 0x192b, 1}, - {0x1930, 0x1931, 1}, - {0x1933, 0x1938, 1}, - {0x19b0, 0x19c0, 1}, - {0x19c8, 0x19c9, 1}, - {0x1a19, 0x1a1b, 1}, - {0x1a55, 0x1a57, 2}, - {0x1a61, 0x1a63, 2}, - {0x1a64, 0x1a6d, 9}, - {0x1a6e, 0x1a72, 1}, - {0x1b04, 0x1b35, 49}, - {0x1b3b, 0x1b3d, 2}, - {0x1b3e, 0x1b41, 1}, - {0x1b43, 0x1b44, 1}, - {0x1b82, 0x1ba1, 31}, - {0x1ba6, 0x1ba7, 1}, - {0x1baa, 0x1be7, 61}, - {0x1bea, 0x1bec, 1}, - {0x1bee, 0x1bf2, 4}, - {0x1bf3, 0x1c24, 49}, - {0x1c25, 0x1c2b, 1}, - {0x1c34, 0x1c35, 1}, - {0x1ce1, 0x1cf2, 17}, - {0xa823, 0xa824, 1}, - {0xa827, 0xa880, 89}, - {0xa881, 0xa8b4, 51}, - {0xa8b5, 0xa8c3, 1}, - {0xa952, 0xa953, 1}, - {0xa983, 0xa9b4, 49}, - {0xa9b5, 0xa9ba, 5}, - {0xa9bb, 0xa9bd, 2}, - {0xa9be, 0xa9c0, 1}, - {0xaa2f, 0xaa30, 1}, - {0xaa33, 0xaa34, 1}, - {0xaa4d, 0xaa7b, 46}, - {0xabe3, 0xabe4, 1}, - {0xabe6, 0xabe7, 1}, - {0xabe9, 0xabea, 1}, - {0xabec, 0x11000, 25620}, - {0x11002, 0x11082, 128}, - {0x110b0, 0x110b2, 1}, - {0x110b7, 0x110b8, 1}, - {0x1d165, 0x1d166, 1}, - {0x1d16d, 0x1d172, 1}, -} - -var _Mn = []Range{ - {0x0300, 0x036f, 1}, - {0x0483, 0x0487, 1}, - {0x0591, 0x05bd, 1}, - {0x05bf, 0x05c1, 2}, - {0x05c2, 0x05c4, 2}, - {0x05c5, 0x05c7, 2}, - {0x0610, 0x061a, 1}, - {0x064b, 0x065f, 1}, - {0x0670, 0x06d6, 102}, - {0x06d7, 0x06dc, 1}, - {0x06df, 0x06e4, 1}, - {0x06e7, 0x06e8, 1}, - {0x06ea, 0x06ed, 1}, - {0x0711, 0x0730, 31}, - {0x0731, 0x074a, 1}, - {0x07a6, 0x07b0, 1}, - {0x07eb, 0x07f3, 1}, - {0x0816, 0x0819, 1}, - {0x081b, 0x0823, 1}, - {0x0825, 0x0827, 1}, - {0x0829, 0x082d, 1}, - {0x0859, 0x085b, 1}, - {0x0900, 0x0902, 1}, - {0x093a, 0x093c, 2}, - {0x0941, 0x0948, 1}, - {0x094d, 0x0951, 4}, - {0x0952, 0x0957, 1}, - {0x0962, 0x0963, 1}, - {0x0981, 0x09bc, 59}, - {0x09c1, 0x09c4, 1}, - {0x09cd, 0x09e2, 21}, - {0x09e3, 0x0a01, 30}, - {0x0a02, 0x0a3c, 58}, - {0x0a41, 0x0a42, 1}, - {0x0a47, 0x0a48, 1}, - {0x0a4b, 0x0a4d, 1}, - {0x0a51, 0x0a70, 31}, - {0x0a71, 0x0a75, 4}, - {0x0a81, 0x0a82, 1}, - {0x0abc, 0x0ac1, 5}, - {0x0ac2, 0x0ac5, 1}, - {0x0ac7, 0x0ac8, 1}, - {0x0acd, 0x0ae2, 21}, - {0x0ae3, 0x0b01, 30}, - {0x0b3c, 0x0b3f, 3}, - {0x0b41, 0x0b44, 1}, - {0x0b4d, 0x0b56, 9}, - {0x0b62, 0x0b63, 1}, - {0x0b82, 0x0bc0, 62}, - {0x0bcd, 0x0c3e, 113}, - {0x0c3f, 0x0c40, 1}, - {0x0c46, 0x0c48, 1}, - {0x0c4a, 0x0c4d, 1}, - {0x0c55, 0x0c56, 1}, - {0x0c62, 0x0c63, 1}, - {0x0cbc, 0x0cbf, 3}, - {0x0cc6, 0x0ccc, 6}, - {0x0ccd, 0x0ce2, 21}, - {0x0ce3, 0x0d41, 94}, - {0x0d42, 0x0d44, 1}, - {0x0d4d, 0x0d62, 21}, - {0x0d63, 0x0dca, 103}, - {0x0dd2, 0x0dd4, 1}, - {0x0dd6, 0x0e31, 91}, - {0x0e34, 0x0e3a, 1}, - {0x0e47, 0x0e4e, 1}, - {0x0eb1, 0x0eb4, 3}, - {0x0eb5, 0x0eb9, 1}, - {0x0ebb, 0x0ebc, 1}, - {0x0ec8, 0x0ecd, 1}, - {0x0f18, 0x0f19, 1}, - {0x0f35, 0x0f39, 2}, - {0x0f71, 0x0f7e, 1}, - {0x0f80, 0x0f84, 1}, - {0x0f86, 0x0f87, 1}, - {0x0f8d, 0x0f97, 1}, - {0x0f99, 0x0fbc, 1}, - {0x0fc6, 0x102d, 103}, - {0x102e, 0x1030, 1}, - {0x1032, 0x1037, 1}, - {0x1039, 0x103a, 1}, - {0x103d, 0x103e, 1}, - {0x1058, 0x1059, 1}, - {0x105e, 0x1060, 1}, - {0x1071, 0x1074, 1}, - {0x1082, 0x1085, 3}, - {0x1086, 0x108d, 7}, - {0x109d, 0x135d, 704}, - {0x135e, 0x135f, 1}, - {0x1712, 0x1714, 1}, - {0x1732, 0x1734, 1}, - {0x1752, 0x1753, 1}, - {0x1772, 0x1773, 1}, - {0x17b7, 0x17bd, 1}, - {0x17c6, 0x17c9, 3}, - {0x17ca, 0x17d3, 1}, - {0x17dd, 0x180b, 46}, - {0x180c, 0x180d, 1}, - {0x18a9, 0x1920, 119}, - {0x1921, 0x1922, 1}, - {0x1927, 0x1928, 1}, - {0x1932, 0x1939, 7}, - {0x193a, 0x193b, 1}, - {0x1a17, 0x1a18, 1}, - {0x1a56, 0x1a58, 2}, - {0x1a59, 0x1a5e, 1}, - {0x1a60, 0x1a62, 2}, - {0x1a65, 0x1a6c, 1}, - {0x1a73, 0x1a7c, 1}, - {0x1a7f, 0x1b00, 129}, - {0x1b01, 0x1b03, 1}, - {0x1b34, 0x1b36, 2}, - {0x1b37, 0x1b3a, 1}, - {0x1b3c, 0x1b42, 6}, - {0x1b6b, 0x1b73, 1}, - {0x1b80, 0x1b81, 1}, - {0x1ba2, 0x1ba5, 1}, - {0x1ba8, 0x1ba9, 1}, - {0x1be6, 0x1be8, 2}, - {0x1be9, 0x1bed, 4}, - {0x1bef, 0x1bf1, 1}, - {0x1c2c, 0x1c33, 1}, - {0x1c36, 0x1c37, 1}, - {0x1cd0, 0x1cd2, 1}, - {0x1cd4, 0x1ce0, 1}, - {0x1ce2, 0x1ce8, 1}, - {0x1ced, 0x1dc0, 211}, - {0x1dc1, 0x1de6, 1}, - {0x1dfc, 0x1dff, 1}, - {0x20d0, 0x20dc, 1}, - {0x20e1, 0x20e5, 4}, - {0x20e6, 0x20f0, 1}, - {0x2cef, 0x2cf1, 1}, - {0x2d7f, 0x2de0, 97}, - {0x2de1, 0x2dff, 1}, - {0x302a, 0x302f, 1}, - {0x3099, 0x309a, 1}, - {0xa66f, 0xa67c, 13}, - {0xa67d, 0xa6f0, 115}, - {0xa6f1, 0xa802, 273}, - {0xa806, 0xa80b, 5}, - {0xa825, 0xa826, 1}, - {0xa8c4, 0xa8e0, 28}, - {0xa8e1, 0xa8f1, 1}, - {0xa926, 0xa92d, 1}, - {0xa947, 0xa951, 1}, - {0xa980, 0xa982, 1}, - {0xa9b3, 0xa9b6, 3}, - {0xa9b7, 0xa9b9, 1}, - {0xa9bc, 0xaa29, 109}, - {0xaa2a, 0xaa2e, 1}, - {0xaa31, 0xaa32, 1}, - {0xaa35, 0xaa36, 1}, - {0xaa43, 0xaa4c, 9}, - {0xaab0, 0xaab2, 2}, - {0xaab3, 0xaab4, 1}, - {0xaab7, 0xaab8, 1}, - {0xaabe, 0xaabf, 1}, - {0xaac1, 0xabe5, 292}, - {0xabe8, 0xabed, 5}, - {0xfb1e, 0xfe00, 738}, - {0xfe01, 0xfe0f, 1}, - {0xfe20, 0xfe26, 1}, - {0x101fd, 0x10a01, 2052}, - {0x10a02, 0x10a03, 1}, - {0x10a05, 0x10a06, 1}, - {0x10a0c, 0x10a0f, 1}, - {0x10a38, 0x10a3a, 1}, - {0x10a3f, 0x11001, 1474}, - {0x11038, 0x11046, 1}, - {0x11080, 0x11081, 1}, - {0x110b3, 0x110b6, 1}, - {0x110b9, 0x110ba, 1}, - {0x1d167, 0x1d169, 1}, - {0x1d17b, 0x1d182, 1}, - {0x1d185, 0x1d18b, 1}, - {0x1d1aa, 0x1d1ad, 1}, - {0x1d242, 0x1d244, 1}, - {0xe0100, 0xe01ef, 1}, -} - -var _Zl = []Range{ - {0x2028, 0x2028, 1}, -} - -var letter = []Range{ - {0x0041, 0x005a, 1}, - {0x0061, 0x007a, 1}, - {0x00aa, 0x00b5, 11}, - {0x00ba, 0x00c0, 6}, - {0x00c1, 0x00d6, 1}, - {0x00d8, 0x00f6, 1}, - {0x00f8, 0x02c1, 1}, - {0x02c6, 0x02d1, 1}, - {0x02e0, 0x02e4, 1}, - {0x02ec, 0x02ee, 2}, - {0x0370, 0x0374, 1}, - {0x0376, 0x0377, 1}, - {0x037a, 0x037d, 1}, - {0x0386, 0x0388, 2}, - {0x0389, 0x038a, 1}, - {0x038c, 0x038e, 2}, - {0x038f, 0x03a1, 1}, - {0x03a3, 0x03f5, 1}, - {0x03f7, 0x0481, 1}, - {0x048a, 0x0527, 1}, - {0x0531, 0x0556, 1}, - {0x0559, 0x0561, 8}, - {0x0562, 0x0587, 1}, - {0x05d0, 0x05ea, 1}, - {0x05f0, 0x05f2, 1}, - {0x0620, 0x064a, 1}, - {0x066e, 0x066f, 1}, - {0x0671, 0x06d3, 1}, - {0x06d5, 0x06e5, 16}, - {0x06e6, 0x06ee, 8}, - {0x06ef, 0x06fa, 11}, - {0x06fb, 0x06fc, 1}, - {0x06ff, 0x0710, 17}, - {0x0712, 0x072f, 1}, - {0x074d, 0x07a5, 1}, - {0x07b1, 0x07ca, 25}, - {0x07cb, 0x07ea, 1}, - {0x07f4, 0x07f5, 1}, - {0x07fa, 0x0800, 6}, - {0x0801, 0x0815, 1}, - {0x081a, 0x0824, 10}, - {0x0828, 0x0840, 24}, - {0x0841, 0x0858, 1}, - {0x0904, 0x0939, 1}, - {0x093d, 0x0950, 19}, - {0x0958, 0x0961, 1}, - {0x0971, 0x0977, 1}, - {0x0979, 0x097f, 1}, - {0x0985, 0x098c, 1}, - {0x098f, 0x0990, 1}, - {0x0993, 0x09a8, 1}, - {0x09aa, 0x09b0, 1}, - {0x09b2, 0x09b6, 4}, - {0x09b7, 0x09b9, 1}, - {0x09bd, 0x09ce, 17}, - {0x09dc, 0x09dd, 1}, - {0x09df, 0x09e1, 1}, - {0x09f0, 0x09f1, 1}, - {0x0a05, 0x0a0a, 1}, - {0x0a0f, 0x0a10, 1}, - {0x0a13, 0x0a28, 1}, - {0x0a2a, 0x0a30, 1}, - {0x0a32, 0x0a33, 1}, - {0x0a35, 0x0a36, 1}, - {0x0a38, 0x0a39, 1}, - {0x0a59, 0x0a5c, 1}, - {0x0a5e, 0x0a72, 20}, - {0x0a73, 0x0a74, 1}, - {0x0a85, 0x0a8d, 1}, - {0x0a8f, 0x0a91, 1}, - {0x0a93, 0x0aa8, 1}, - {0x0aaa, 0x0ab0, 1}, - {0x0ab2, 0x0ab3, 1}, - {0x0ab5, 0x0ab9, 1}, - {0x0abd, 0x0ad0, 19}, - {0x0ae0, 0x0ae1, 1}, - {0x0b05, 0x0b0c, 1}, - {0x0b0f, 0x0b10, 1}, - {0x0b13, 0x0b28, 1}, - {0x0b2a, 0x0b30, 1}, - {0x0b32, 0x0b33, 1}, - {0x0b35, 0x0b39, 1}, - {0x0b3d, 0x0b5c, 31}, - {0x0b5d, 0x0b5f, 2}, - {0x0b60, 0x0b61, 1}, - {0x0b71, 0x0b83, 18}, - {0x0b85, 0x0b8a, 1}, - {0x0b8e, 0x0b90, 1}, - {0x0b92, 0x0b95, 1}, - {0x0b99, 0x0b9a, 1}, - {0x0b9c, 0x0b9e, 2}, - {0x0b9f, 0x0ba3, 4}, - {0x0ba4, 0x0ba8, 4}, - {0x0ba9, 0x0baa, 1}, - {0x0bae, 0x0bb9, 1}, - {0x0bd0, 0x0c05, 53}, - {0x0c06, 0x0c0c, 1}, - {0x0c0e, 0x0c10, 1}, - {0x0c12, 0x0c28, 1}, - {0x0c2a, 0x0c33, 1}, - {0x0c35, 0x0c39, 1}, - {0x0c3d, 0x0c58, 27}, - {0x0c59, 0x0c60, 7}, - {0x0c61, 0x0c85, 36}, - {0x0c86, 0x0c8c, 1}, - {0x0c8e, 0x0c90, 1}, - {0x0c92, 0x0ca8, 1}, - {0x0caa, 0x0cb3, 1}, - {0x0cb5, 0x0cb9, 1}, - {0x0cbd, 0x0cde, 33}, - {0x0ce0, 0x0ce1, 1}, - {0x0cf1, 0x0cf2, 1}, - {0x0d05, 0x0d0c, 1}, - {0x0d0e, 0x0d10, 1}, - {0x0d12, 0x0d3a, 1}, - {0x0d3d, 0x0d4e, 17}, - {0x0d60, 0x0d61, 1}, - {0x0d7a, 0x0d7f, 1}, - {0x0d85, 0x0d96, 1}, - {0x0d9a, 0x0db1, 1}, - {0x0db3, 0x0dbb, 1}, - {0x0dbd, 0x0dc0, 3}, - {0x0dc1, 0x0dc6, 1}, - {0x0e01, 0x0e30, 1}, - {0x0e32, 0x0e33, 1}, - {0x0e40, 0x0e46, 1}, - {0x0e81, 0x0e82, 1}, - {0x0e84, 0x0e87, 3}, - {0x0e88, 0x0e8a, 2}, - {0x0e8d, 0x0e94, 7}, - {0x0e95, 0x0e97, 1}, - {0x0e99, 0x0e9f, 1}, - {0x0ea1, 0x0ea3, 1}, - {0x0ea5, 0x0ea7, 2}, - {0x0eaa, 0x0eab, 1}, - {0x0ead, 0x0eb0, 1}, - {0x0eb2, 0x0eb3, 1}, - {0x0ebd, 0x0ec0, 3}, - {0x0ec1, 0x0ec4, 1}, - {0x0ec6, 0x0edc, 22}, - {0x0edd, 0x0f00, 35}, - {0x0f40, 0x0f47, 1}, - {0x0f49, 0x0f6c, 1}, - {0x0f88, 0x0f8c, 1}, - {0x1000, 0x102a, 1}, - {0x103f, 0x1050, 17}, - {0x1051, 0x1055, 1}, - {0x105a, 0x105d, 1}, - {0x1061, 0x1065, 4}, - {0x1066, 0x106e, 8}, - {0x106f, 0x1070, 1}, - {0x1075, 0x1081, 1}, - {0x108e, 0x10a0, 18}, - {0x10a1, 0x10c5, 1}, - {0x10d0, 0x10fa, 1}, - {0x10fc, 0x1100, 4}, - {0x1101, 0x1248, 1}, - {0x124a, 0x124d, 1}, - {0x1250, 0x1256, 1}, - {0x1258, 0x125a, 2}, - {0x125b, 0x125d, 1}, - {0x1260, 0x1288, 1}, - {0x128a, 0x128d, 1}, - {0x1290, 0x12b0, 1}, - {0x12b2, 0x12b5, 1}, - {0x12b8, 0x12be, 1}, - {0x12c0, 0x12c2, 2}, - {0x12c3, 0x12c5, 1}, - {0x12c8, 0x12d6, 1}, - {0x12d8, 0x1310, 1}, - {0x1312, 0x1315, 1}, - {0x1318, 0x135a, 1}, - {0x1380, 0x138f, 1}, - {0x13a0, 0x13f4, 1}, - {0x1401, 0x166c, 1}, - {0x166f, 0x167f, 1}, - {0x1681, 0x169a, 1}, - {0x16a0, 0x16ea, 1}, - {0x1700, 0x170c, 1}, - {0x170e, 0x1711, 1}, - {0x1720, 0x1731, 1}, - {0x1740, 0x1751, 1}, - {0x1760, 0x176c, 1}, - {0x176e, 0x1770, 1}, - {0x1780, 0x17b3, 1}, - {0x17d7, 0x17dc, 5}, - {0x1820, 0x1877, 1}, - {0x1880, 0x18a8, 1}, - {0x18aa, 0x18b0, 6}, - {0x18b1, 0x18f5, 1}, - {0x1900, 0x191c, 1}, - {0x1950, 0x196d, 1}, - {0x1970, 0x1974, 1}, - {0x1980, 0x19ab, 1}, - {0x19c1, 0x19c7, 1}, - {0x1a00, 0x1a16, 1}, - {0x1a20, 0x1a54, 1}, - {0x1aa7, 0x1b05, 94}, - {0x1b06, 0x1b33, 1}, - {0x1b45, 0x1b4b, 1}, - {0x1b83, 0x1ba0, 1}, - {0x1bae, 0x1baf, 1}, - {0x1bc0, 0x1be5, 1}, - {0x1c00, 0x1c23, 1}, - {0x1c4d, 0x1c4f, 1}, - {0x1c5a, 0x1c7d, 1}, - {0x1ce9, 0x1cec, 1}, - {0x1cee, 0x1cf1, 1}, - {0x1d00, 0x1dbf, 1}, - {0x1e00, 0x1f15, 1}, - {0x1f18, 0x1f1d, 1}, - {0x1f20, 0x1f45, 1}, - {0x1f48, 0x1f4d, 1}, - {0x1f50, 0x1f57, 1}, - {0x1f59, 0x1f5f, 2}, - {0x1f60, 0x1f7d, 1}, - {0x1f80, 0x1fb4, 1}, - {0x1fb6, 0x1fbc, 1}, - {0x1fbe, 0x1fc2, 4}, - {0x1fc3, 0x1fc4, 1}, - {0x1fc6, 0x1fcc, 1}, - {0x1fd0, 0x1fd3, 1}, - {0x1fd6, 0x1fdb, 1}, - {0x1fe0, 0x1fec, 1}, - {0x1ff2, 0x1ff4, 1}, - {0x1ff6, 0x1ffc, 1}, - {0x2071, 0x207f, 14}, - {0x2090, 0x209c, 1}, - {0x2102, 0x2107, 5}, - {0x210a, 0x2113, 1}, - {0x2115, 0x2119, 4}, - {0x211a, 0x211d, 1}, - {0x2124, 0x212a, 2}, - {0x212b, 0x212d, 1}, - {0x212f, 0x2139, 1}, - {0x213c, 0x213f, 1}, - {0x2145, 0x2149, 1}, - {0x214e, 0x2183, 53}, - {0x2184, 0x2c00, 2684}, - {0x2c01, 0x2c2e, 1}, - {0x2c30, 0x2c5e, 1}, - {0x2c60, 0x2ce4, 1}, - {0x2ceb, 0x2cee, 1}, - {0x2d00, 0x2d25, 1}, - {0x2d30, 0x2d65, 1}, - {0x2d6f, 0x2d80, 17}, - {0x2d81, 0x2d96, 1}, - {0x2da0, 0x2da6, 1}, - {0x2da8, 0x2dae, 1}, - {0x2db0, 0x2db6, 1}, - {0x2db8, 0x2dbe, 1}, - {0x2dc0, 0x2dc6, 1}, - {0x2dc8, 0x2dce, 1}, - {0x2dd0, 0x2dd6, 1}, - {0x2dd8, 0x2dde, 1}, - {0x2e2f, 0x3005, 470}, - {0x3006, 0x3031, 43}, - {0x3032, 0x3035, 1}, - {0x303b, 0x303c, 1}, - {0x3041, 0x3096, 1}, - {0x309d, 0x309f, 1}, - {0x30a1, 0x30fa, 1}, - {0x30fc, 0x30ff, 1}, - {0x3105, 0x312d, 1}, - {0x3131, 0x318e, 1}, - {0x31a0, 0x31ba, 1}, - {0x31f0, 0x31ff, 1}, - {0x3400, 0x4db5, 1}, - {0x4e00, 0x9fcb, 1}, - {0xa000, 0xa48c, 1}, - {0xa4d0, 0xa4fd, 1}, - {0xa500, 0xa60c, 1}, - {0xa610, 0xa61f, 1}, - {0xa62a, 0xa62b, 1}, - {0xa640, 0xa66e, 1}, - {0xa67f, 0xa697, 1}, - {0xa6a0, 0xa6e5, 1}, - {0xa717, 0xa71f, 1}, - {0xa722, 0xa788, 1}, - {0xa78b, 0xa78e, 1}, - {0xa790, 0xa791, 1}, - {0xa7a0, 0xa7a9, 1}, - {0xa7fa, 0xa801, 1}, - {0xa803, 0xa805, 1}, - {0xa807, 0xa80a, 1}, - {0xa80c, 0xa822, 1}, - {0xa840, 0xa873, 1}, - {0xa882, 0xa8b3, 1}, - {0xa8f2, 0xa8f7, 1}, - {0xa8fb, 0xa90a, 15}, - {0xa90b, 0xa925, 1}, - {0xa930, 0xa946, 1}, - {0xa960, 0xa97c, 1}, - {0xa984, 0xa9b2, 1}, - {0xa9cf, 0xaa00, 49}, - {0xaa01, 0xaa28, 1}, - {0xaa40, 0xaa42, 1}, - {0xaa44, 0xaa4b, 1}, - {0xaa60, 0xaa76, 1}, - {0xaa7a, 0xaa80, 6}, - {0xaa81, 0xaaaf, 1}, - {0xaab1, 0xaab5, 4}, - {0xaab6, 0xaab9, 3}, - {0xaaba, 0xaabd, 1}, - {0xaac0, 0xaac2, 2}, - {0xaadb, 0xaadd, 1}, - {0xab01, 0xab06, 1}, - {0xab09, 0xab0e, 1}, - {0xab11, 0xab16, 1}, - {0xab20, 0xab26, 1}, - {0xab28, 0xab2e, 1}, - {0xabc0, 0xabe2, 1}, - {0xac00, 0xd7a3, 1}, - {0xd7b0, 0xd7c6, 1}, - {0xd7cb, 0xd7fb, 1}, - {0xf900, 0xfa2d, 1}, - {0xfa30, 0xfa6d, 1}, - {0xfa70, 0xfad9, 1}, - {0xfb00, 0xfb06, 1}, - {0xfb13, 0xfb17, 1}, - {0xfb1d, 0xfb1f, 2}, - {0xfb20, 0xfb28, 1}, - {0xfb2a, 0xfb36, 1}, - {0xfb38, 0xfb3c, 1}, - {0xfb3e, 0xfb40, 2}, - {0xfb41, 0xfb43, 2}, - {0xfb44, 0xfb46, 2}, - {0xfb47, 0xfbb1, 1}, - {0xfbd3, 0xfd3d, 1}, - {0xfd50, 0xfd8f, 1}, - {0xfd92, 0xfdc7, 1}, - {0xfdf0, 0xfdfb, 1}, - {0xfe70, 0xfe74, 1}, - {0xfe76, 0xfefc, 1}, - {0xff21, 0xff3a, 1}, - {0xff41, 0xff5a, 1}, - {0xff66, 0xffbe, 1}, - {0xffc2, 0xffc7, 1}, - {0xffca, 0xffcf, 1}, - {0xffd2, 0xffd7, 1}, - {0xffda, 0xffdc, 1}, - {0x10000, 0x1000b, 1}, - {0x1000d, 0x10026, 1}, - {0x10028, 0x1003a, 1}, - {0x1003c, 0x1003d, 1}, - {0x1003f, 0x1004d, 1}, - {0x10050, 0x1005d, 1}, - {0x10080, 0x100fa, 1}, - {0x10280, 0x1029c, 1}, - {0x102a0, 0x102d0, 1}, - {0x10300, 0x1031e, 1}, - {0x10330, 0x10340, 1}, - {0x10342, 0x10349, 1}, - {0x10380, 0x1039d, 1}, - {0x103a0, 0x103c3, 1}, - {0x103c8, 0x103cf, 1}, - {0x10400, 0x1049d, 1}, - {0x10800, 0x10805, 1}, - {0x10808, 0x1080a, 2}, - {0x1080b, 0x10835, 1}, - {0x10837, 0x10838, 1}, - {0x1083c, 0x1083f, 3}, - {0x10840, 0x10855, 1}, - {0x10900, 0x10915, 1}, - {0x10920, 0x10939, 1}, - {0x10a00, 0x10a10, 16}, - {0x10a11, 0x10a13, 1}, - {0x10a15, 0x10a17, 1}, - {0x10a19, 0x10a33, 1}, - {0x10a60, 0x10a7c, 1}, - {0x10b00, 0x10b35, 1}, - {0x10b40, 0x10b55, 1}, - {0x10b60, 0x10b72, 1}, - {0x10c00, 0x10c48, 1}, - {0x11003, 0x11037, 1}, - {0x11083, 0x110af, 1}, - {0x12000, 0x1236e, 1}, - {0x13000, 0x1342e, 1}, - {0x16800, 0x16a38, 1}, - {0x1b000, 0x1b001, 1}, - {0x1d400, 0x1d454, 1}, - {0x1d456, 0x1d49c, 1}, - {0x1d49e, 0x1d49f, 1}, - {0x1d4a2, 0x1d4a5, 3}, - {0x1d4a6, 0x1d4a9, 3}, - {0x1d4aa, 0x1d4ac, 1}, - {0x1d4ae, 0x1d4b9, 1}, - {0x1d4bb, 0x1d4bd, 2}, - {0x1d4be, 0x1d4c3, 1}, - {0x1d4c5, 0x1d505, 1}, - {0x1d507, 0x1d50a, 1}, - {0x1d50d, 0x1d514, 1}, - {0x1d516, 0x1d51c, 1}, - {0x1d51e, 0x1d539, 1}, - {0x1d53b, 0x1d53e, 1}, - {0x1d540, 0x1d544, 1}, - {0x1d546, 0x1d54a, 4}, - {0x1d54b, 0x1d550, 1}, - {0x1d552, 0x1d6a5, 1}, - {0x1d6a8, 0x1d6c0, 1}, - {0x1d6c2, 0x1d6da, 1}, - {0x1d6dc, 0x1d6fa, 1}, - {0x1d6fc, 0x1d714, 1}, - {0x1d716, 0x1d734, 1}, - {0x1d736, 0x1d74e, 1}, - {0x1d750, 0x1d76e, 1}, - {0x1d770, 0x1d788, 1}, - {0x1d78a, 0x1d7a8, 1}, - {0x1d7aa, 0x1d7c2, 1}, - {0x1d7c4, 0x1d7cb, 1}, - {0x20000, 0x2a6d6, 1}, - {0x2a700, 0x2b734, 1}, - {0x2b740, 0x2b81d, 1}, - {0x2f800, 0x2fa1d, 1}, -} - -var _Zp = []Range{ - {0x2029, 0x2029, 1}, -} - -var _Zs = []Range{ - {0x0020, 0x00a0, 128}, - {0x1680, 0x180e, 398}, - {0x2000, 0x200a, 1}, - {0x202f, 0x205f, 48}, - {0x3000, 0x3000, 1}, -} - -var _Cs = []Range{ - {0xd800, 0xdfff, 1}, -} - -var _Co = []Range{ - {0xe000, 0xf8ff, 1}, - {0xf0000, 0xffffd, 1}, - {0x100000, 0x10fffd, 1}, -} - -var _Cf = []Range{ - {0x00ad, 0x0600, 1363}, - {0x0601, 0x0603, 1}, - {0x06dd, 0x070f, 50}, - {0x17b4, 0x17b5, 1}, - {0x200b, 0x200f, 1}, - {0x202a, 0x202e, 1}, - {0x2060, 0x2064, 1}, - {0x206a, 0x206f, 1}, - {0xfeff, 0xfff9, 250}, - {0xfffa, 0xfffb, 1}, - {0x110bd, 0x1d173, 49334}, - {0x1d174, 0x1d17a, 1}, - {0xe0001, 0xe0020, 31}, - {0xe0021, 0xe007f, 1}, -} - -var _Cc = []Range{ - {0x0001, 0x001f, 1}, - {0x007f, 0x009f, 1}, -} - -var _Po = []Range{ - {0x0021, 0x0023, 1}, - {0x0025, 0x0027, 1}, - {0x002a, 0x002e, 2}, - {0x002f, 0x003a, 11}, - {0x003b, 0x003f, 4}, - {0x0040, 0x005c, 28}, - {0x00a1, 0x00b7, 22}, - {0x00bf, 0x037e, 703}, - {0x0387, 0x055a, 467}, - {0x055b, 0x055f, 1}, - {0x0589, 0x05c0, 55}, - {0x05c3, 0x05c6, 3}, - {0x05f3, 0x05f4, 1}, - {0x0609, 0x060a, 1}, - {0x060c, 0x060d, 1}, - {0x061b, 0x061e, 3}, - {0x061f, 0x066a, 75}, - {0x066b, 0x066d, 1}, - {0x06d4, 0x0700, 44}, - {0x0701, 0x070d, 1}, - {0x07f7, 0x07f9, 1}, - {0x0830, 0x083e, 1}, - {0x085e, 0x0964, 262}, - {0x0965, 0x0970, 11}, - {0x0df4, 0x0e4f, 91}, - {0x0e5a, 0x0e5b, 1}, - {0x0f04, 0x0f12, 1}, - {0x0f85, 0x0fd0, 75}, - {0x0fd1, 0x0fd4, 1}, - {0x0fd9, 0x0fda, 1}, - {0x104a, 0x104f, 1}, - {0x10fb, 0x1361, 614}, - {0x1362, 0x1368, 1}, - {0x166d, 0x166e, 1}, - {0x16eb, 0x16ed, 1}, - {0x1735, 0x1736, 1}, - {0x17d4, 0x17d6, 1}, - {0x17d8, 0x17da, 1}, - {0x1800, 0x1805, 1}, - {0x1807, 0x180a, 1}, - {0x1944, 0x1945, 1}, - {0x1a1e, 0x1a1f, 1}, - {0x1aa0, 0x1aa6, 1}, - {0x1aa8, 0x1aad, 1}, - {0x1b5a, 0x1b60, 1}, - {0x1bfc, 0x1bff, 1}, - {0x1c3b, 0x1c3f, 1}, - {0x1c7e, 0x1c7f, 1}, - {0x1cd3, 0x2016, 835}, - {0x2017, 0x2020, 9}, - {0x2021, 0x2027, 1}, - {0x2030, 0x2038, 1}, - {0x203b, 0x203e, 1}, - {0x2041, 0x2043, 1}, - {0x2047, 0x2051, 1}, - {0x2053, 0x2055, 2}, - {0x2056, 0x205e, 1}, - {0x2cf9, 0x2cfc, 1}, - {0x2cfe, 0x2cff, 1}, - {0x2d70, 0x2e00, 144}, - {0x2e01, 0x2e06, 5}, - {0x2e07, 0x2e08, 1}, - {0x2e0b, 0x2e0e, 3}, - {0x2e0f, 0x2e16, 1}, - {0x2e18, 0x2e19, 1}, - {0x2e1b, 0x2e1e, 3}, - {0x2e1f, 0x2e2a, 11}, - {0x2e2b, 0x2e2e, 1}, - {0x2e30, 0x2e31, 1}, - {0x3001, 0x3003, 1}, - {0x303d, 0x30fb, 190}, - {0xa4fe, 0xa4ff, 1}, - {0xa60d, 0xa60f, 1}, - {0xa673, 0xa67e, 11}, - {0xa6f2, 0xa6f7, 1}, - {0xa874, 0xa877, 1}, - {0xa8ce, 0xa8cf, 1}, - {0xa8f8, 0xa8fa, 1}, - {0xa92e, 0xa92f, 1}, - {0xa95f, 0xa9c1, 98}, - {0xa9c2, 0xa9cd, 1}, - {0xa9de, 0xa9df, 1}, - {0xaa5c, 0xaa5f, 1}, - {0xaade, 0xaadf, 1}, - {0xabeb, 0xfe10, 21029}, - {0xfe11, 0xfe16, 1}, - {0xfe19, 0xfe30, 23}, - {0xfe45, 0xfe46, 1}, - {0xfe49, 0xfe4c, 1}, - {0xfe50, 0xfe52, 1}, - {0xfe54, 0xfe57, 1}, - {0xfe5f, 0xfe61, 1}, - {0xfe68, 0xfe6a, 2}, - {0xfe6b, 0xff01, 150}, - {0xff02, 0xff03, 1}, - {0xff05, 0xff07, 1}, - {0xff0a, 0xff0e, 2}, - {0xff0f, 0xff1a, 11}, - {0xff1b, 0xff1f, 4}, - {0xff20, 0xff3c, 28}, - {0xff61, 0xff64, 3}, - {0xff65, 0x10100, 411}, - {0x10101, 0x1039f, 670}, - {0x103d0, 0x10857, 1159}, - {0x1091f, 0x1093f, 32}, - {0x10a50, 0x10a58, 1}, - {0x10a7f, 0x10b39, 186}, - {0x10b3a, 0x10b3f, 1}, - {0x11047, 0x1104d, 1}, - {0x110bb, 0x110bc, 1}, - {0x110be, 0x110c1, 1}, - {0x12470, 0x12473, 1}, -} - -var _Pi = []Range{ - {0x00ab, 0x2018, 8045}, - {0x201b, 0x201c, 1}, - {0x201f, 0x2039, 26}, - {0x2e02, 0x2e04, 2}, - {0x2e09, 0x2e0c, 3}, - {0x2e1c, 0x2e20, 4}, -} - -var _Pf = []Range{ - {0x00bb, 0x2019, 8030}, - {0x201d, 0x203a, 29}, - {0x2e03, 0x2e05, 2}, - {0x2e0a, 0x2e0d, 3}, - {0x2e1d, 0x2e21, 4}, -} - -var _Pe = []Range{ - {0x0029, 0x005d, 52}, - {0x007d, 0x0f3b, 3774}, - {0x0f3d, 0x169c, 1887}, - {0x2046, 0x207e, 56}, - {0x208e, 0x232a, 668}, - {0x2769, 0x2775, 2}, - {0x27c6, 0x27e7, 33}, - {0x27e9, 0x27ef, 2}, - {0x2984, 0x2998, 2}, - {0x29d9, 0x29db, 2}, - {0x29fd, 0x2e23, 1062}, - {0x2e25, 0x2e29, 2}, - {0x3009, 0x3011, 2}, - {0x3015, 0x301b, 2}, - {0x301e, 0x301f, 1}, - {0xfd3f, 0xfe18, 217}, - {0xfe36, 0xfe44, 2}, - {0xfe48, 0xfe5a, 18}, - {0xfe5c, 0xfe5e, 2}, - {0xff09, 0xff3d, 52}, - {0xff5d, 0xff63, 3}, -} - -var _Pd = []Range{ - {0x002d, 0x058a, 1373}, - {0x05be, 0x1400, 3650}, - {0x1806, 0x2010, 2058}, - {0x2011, 0x2015, 1}, - {0x2e17, 0x2e1a, 3}, - {0x301c, 0x3030, 20}, - {0x30a0, 0xfe31, 52625}, - {0xfe32, 0xfe58, 38}, - {0xfe63, 0xff0d, 170}, -} - -var _Pc = []Range{ - {0x005f, 0x203f, 8160}, - {0x2040, 0x2054, 20}, - {0xfe33, 0xfe34, 1}, - {0xfe4d, 0xfe4f, 1}, - {0xff3f, 0xff3f, 1}, -} - -var _Ps = []Range{ - {0x0028, 0x005b, 51}, - {0x007b, 0x0f3a, 3775}, - {0x0f3c, 0x169b, 1887}, - {0x201a, 0x201e, 4}, - {0x2045, 0x207d, 56}, - {0x208d, 0x2329, 668}, - {0x2768, 0x2774, 2}, - {0x27c5, 0x27e6, 33}, - {0x27e8, 0x27ee, 2}, - {0x2983, 0x2997, 2}, - {0x29d8, 0x29da, 2}, - {0x29fc, 0x2e22, 1062}, - {0x2e24, 0x2e28, 2}, - {0x3008, 0x3010, 2}, - {0x3014, 0x301a, 2}, - {0x301d, 0xfd3e, 52513}, - {0xfe17, 0xfe35, 30}, - {0xfe37, 0xfe43, 2}, - {0xfe47, 0xfe59, 18}, - {0xfe5b, 0xfe5d, 2}, - {0xff08, 0xff3b, 51}, - {0xff5b, 0xff5f, 4}, - {0xff62, 0xff62, 1}, -} - -var _Nd = []Range{ - {0x0030, 0x0039, 1}, - {0x0660, 0x0669, 1}, - {0x06f0, 0x06f9, 1}, - {0x07c0, 0x07c9, 1}, - {0x0966, 0x096f, 1}, - {0x09e6, 0x09ef, 1}, - {0x0a66, 0x0a6f, 1}, - {0x0ae6, 0x0aef, 1}, - {0x0b66, 0x0b6f, 1}, - {0x0be6, 0x0bef, 1}, - {0x0c66, 0x0c6f, 1}, - {0x0ce6, 0x0cef, 1}, - {0x0d66, 0x0d6f, 1}, - {0x0e50, 0x0e59, 1}, - {0x0ed0, 0x0ed9, 1}, - {0x0f20, 0x0f29, 1}, - {0x1040, 0x1049, 1}, - {0x1090, 0x1099, 1}, - {0x17e0, 0x17e9, 1}, - {0x1810, 0x1819, 1}, - {0x1946, 0x194f, 1}, - {0x19d0, 0x19d9, 1}, - {0x1a80, 0x1a89, 1}, - {0x1a90, 0x1a99, 1}, - {0x1b50, 0x1b59, 1}, - {0x1bb0, 0x1bb9, 1}, - {0x1c40, 0x1c49, 1}, - {0x1c50, 0x1c59, 1}, - {0xa620, 0xa629, 1}, - {0xa8d0, 0xa8d9, 1}, - {0xa900, 0xa909, 1}, - {0xa9d0, 0xa9d9, 1}, - {0xaa50, 0xaa59, 1}, - {0xabf0, 0xabf9, 1}, - {0xff10, 0xff19, 1}, - {0x104a0, 0x104a9, 1}, - {0x11066, 0x1106f, 1}, - {0x1d7ce, 0x1d7ff, 1}, -} - -var _Nl = []Range{ - {0x16ee, 0x16f0, 1}, - {0x2160, 0x2182, 1}, - {0x2185, 0x2188, 1}, - {0x3007, 0x3021, 26}, - {0x3022, 0x3029, 1}, - {0x3038, 0x303a, 1}, - {0xa6e6, 0xa6ef, 1}, - {0x10140, 0x10174, 1}, - {0x10341, 0x1034a, 9}, - {0x103d1, 0x103d5, 1}, - {0x12400, 0x12462, 1}, -} - -var _No = []Range{ - {0x00b2, 0x00b3, 1}, - {0x00b9, 0x00bc, 3}, - {0x00bd, 0x00be, 1}, - {0x09f4, 0x09f9, 1}, - {0x0b72, 0x0b77, 1}, - {0x0bf0, 0x0bf2, 1}, - {0x0c78, 0x0c7e, 1}, - {0x0d70, 0x0d75, 1}, - {0x0f2a, 0x0f33, 1}, - {0x1369, 0x137c, 1}, - {0x17f0, 0x17f9, 1}, - {0x19da, 0x2070, 1686}, - {0x2074, 0x2079, 1}, - {0x2080, 0x2089, 1}, - {0x2150, 0x215f, 1}, - {0x2189, 0x2460, 727}, - {0x2461, 0x249b, 1}, - {0x24ea, 0x24ff, 1}, - {0x2776, 0x2793, 1}, - {0x2cfd, 0x3192, 1173}, - {0x3193, 0x3195, 1}, - {0x3220, 0x3229, 1}, - {0x3251, 0x325f, 1}, - {0x3280, 0x3289, 1}, - {0x32b1, 0x32bf, 1}, - {0xa830, 0xa835, 1}, - {0x10107, 0x10133, 1}, - {0x10175, 0x10178, 1}, - {0x1018a, 0x10320, 406}, - {0x10321, 0x10323, 1}, - {0x10858, 0x1085f, 1}, - {0x10916, 0x1091b, 1}, - {0x10a40, 0x10a47, 1}, - {0x10a7d, 0x10a7e, 1}, - {0x10b58, 0x10b5f, 1}, - {0x10b78, 0x10b7f, 1}, - {0x10e60, 0x10e7e, 1}, - {0x11052, 0x11065, 1}, - {0x1d360, 0x1d371, 1}, - {0x1f100, 0x1f10a, 1}, -} - -var _So = []Range{ - {0x00a6, 0x00a7, 1}, - {0x00a9, 0x00ae, 5}, - {0x00b0, 0x00b6, 6}, - {0x0482, 0x060e, 396}, - {0x060f, 0x06de, 207}, - {0x06e9, 0x06fd, 20}, - {0x06fe, 0x07f6, 248}, - {0x09fa, 0x0b70, 374}, - {0x0bf3, 0x0bf8, 1}, - {0x0bfa, 0x0c7f, 133}, - {0x0d79, 0x0f01, 392}, - {0x0f02, 0x0f03, 1}, - {0x0f13, 0x0f17, 1}, - {0x0f1a, 0x0f1f, 1}, - {0x0f34, 0x0f38, 2}, - {0x0fbe, 0x0fc5, 1}, - {0x0fc7, 0x0fcc, 1}, - {0x0fce, 0x0fcf, 1}, - {0x0fd5, 0x0fd8, 1}, - {0x109e, 0x109f, 1}, - {0x1360, 0x1390, 48}, - {0x1391, 0x1399, 1}, - {0x1940, 0x19de, 158}, - {0x19df, 0x19ff, 1}, - {0x1b61, 0x1b6a, 1}, - {0x1b74, 0x1b7c, 1}, - {0x2100, 0x2101, 1}, - {0x2103, 0x2106, 1}, - {0x2108, 0x2109, 1}, - {0x2114, 0x2116, 2}, - {0x2117, 0x211e, 7}, - {0x211f, 0x2123, 1}, - {0x2125, 0x2129, 2}, - {0x212e, 0x213a, 12}, - {0x213b, 0x214a, 15}, - {0x214c, 0x214d, 1}, - {0x214f, 0x2195, 70}, - {0x2196, 0x2199, 1}, - {0x219c, 0x219f, 1}, - {0x21a1, 0x21a2, 1}, - {0x21a4, 0x21a5, 1}, - {0x21a7, 0x21ad, 1}, - {0x21af, 0x21cd, 1}, - {0x21d0, 0x21d1, 1}, - {0x21d3, 0x21d5, 2}, - {0x21d6, 0x21f3, 1}, - {0x2300, 0x2307, 1}, - {0x230c, 0x231f, 1}, - {0x2322, 0x2328, 1}, - {0x232b, 0x237b, 1}, - {0x237d, 0x239a, 1}, - {0x23b4, 0x23db, 1}, - {0x23e2, 0x23f3, 1}, - {0x2400, 0x2426, 1}, - {0x2440, 0x244a, 1}, - {0x249c, 0x24e9, 1}, - {0x2500, 0x25b6, 1}, - {0x25b8, 0x25c0, 1}, - {0x25c2, 0x25f7, 1}, - {0x2600, 0x266e, 1}, - {0x2670, 0x26ff, 1}, - {0x2701, 0x2767, 1}, - {0x2794, 0x27bf, 1}, - {0x2800, 0x28ff, 1}, - {0x2b00, 0x2b2f, 1}, - {0x2b45, 0x2b46, 1}, - {0x2b50, 0x2b59, 1}, - {0x2ce5, 0x2cea, 1}, - {0x2e80, 0x2e99, 1}, - {0x2e9b, 0x2ef3, 1}, - {0x2f00, 0x2fd5, 1}, - {0x2ff0, 0x2ffb, 1}, - {0x3004, 0x3012, 14}, - {0x3013, 0x3020, 13}, - {0x3036, 0x3037, 1}, - {0x303e, 0x303f, 1}, - {0x3190, 0x3191, 1}, - {0x3196, 0x319f, 1}, - {0x31c0, 0x31e3, 1}, - {0x3200, 0x321e, 1}, - {0x322a, 0x3250, 1}, - {0x3260, 0x327f, 1}, - {0x328a, 0x32b0, 1}, - {0x32c0, 0x32fe, 1}, - {0x3300, 0x33ff, 1}, - {0x4dc0, 0x4dff, 1}, - {0xa490, 0xa4c6, 1}, - {0xa828, 0xa82b, 1}, - {0xa836, 0xa837, 1}, - {0xa839, 0xaa77, 574}, - {0xaa78, 0xaa79, 1}, - {0xfdfd, 0xffe4, 487}, - {0xffe8, 0xffed, 5}, - {0xffee, 0xfffc, 14}, - {0xfffd, 0x10102, 261}, - {0x10137, 0x1013f, 1}, - {0x10179, 0x10189, 1}, - {0x10190, 0x1019b, 1}, - {0x101d0, 0x101fc, 1}, - {0x1d000, 0x1d0f5, 1}, - {0x1d100, 0x1d126, 1}, - {0x1d129, 0x1d164, 1}, - {0x1d16a, 0x1d16c, 1}, - {0x1d183, 0x1d184, 1}, - {0x1d18c, 0x1d1a9, 1}, - {0x1d1ae, 0x1d1dd, 1}, - {0x1d200, 0x1d241, 1}, - {0x1d245, 0x1d300, 187}, - {0x1d301, 0x1d356, 1}, - {0x1f000, 0x1f02b, 1}, - {0x1f030, 0x1f093, 1}, - {0x1f0a0, 0x1f0ae, 1}, - {0x1f0b1, 0x1f0be, 1}, - {0x1f0c1, 0x1f0cf, 1}, - {0x1f0d1, 0x1f0df, 1}, - {0x1f110, 0x1f12e, 1}, - {0x1f130, 0x1f169, 1}, - {0x1f170, 0x1f19a, 1}, - {0x1f1e6, 0x1f202, 1}, - {0x1f210, 0x1f23a, 1}, - {0x1f240, 0x1f248, 1}, - {0x1f250, 0x1f251, 1}, - {0x1f300, 0x1f320, 1}, - {0x1f330, 0x1f335, 1}, - {0x1f337, 0x1f37c, 1}, - {0x1f380, 0x1f393, 1}, - {0x1f3a0, 0x1f3c4, 1}, - {0x1f3c6, 0x1f3ca, 1}, - {0x1f3e0, 0x1f3f0, 1}, - {0x1f400, 0x1f43e, 1}, - {0x1f440, 0x1f442, 2}, - {0x1f443, 0x1f4f7, 1}, - {0x1f4f9, 0x1f4fc, 1}, - {0x1f500, 0x1f53d, 1}, - {0x1f550, 0x1f567, 1}, - {0x1f5fb, 0x1f5ff, 1}, - {0x1f601, 0x1f610, 1}, - {0x1f612, 0x1f614, 1}, - {0x1f616, 0x1f61c, 2}, - {0x1f61d, 0x1f61e, 1}, - {0x1f620, 0x1f625, 1}, - {0x1f628, 0x1f62b, 1}, - {0x1f62d, 0x1f630, 3}, - {0x1f631, 0x1f633, 1}, - {0x1f635, 0x1f640, 1}, - {0x1f645, 0x1f64f, 1}, - {0x1f680, 0x1f6c5, 1}, - {0x1f700, 0x1f773, 1}, -} - -var _Sm = []Range{ - {0x002b, 0x003c, 17}, - {0x003d, 0x003e, 1}, - {0x007c, 0x007e, 2}, - {0x00ac, 0x00b1, 5}, - {0x00d7, 0x00f7, 32}, - {0x03f6, 0x0606, 528}, - {0x0607, 0x0608, 1}, - {0x2044, 0x2052, 14}, - {0x207a, 0x207c, 1}, - {0x208a, 0x208c, 1}, - {0x2118, 0x2140, 40}, - {0x2141, 0x2144, 1}, - {0x214b, 0x2190, 69}, - {0x2191, 0x2194, 1}, - {0x219a, 0x219b, 1}, - {0x21a0, 0x21a6, 3}, - {0x21ae, 0x21ce, 32}, - {0x21cf, 0x21d2, 3}, - {0x21d4, 0x21f4, 32}, - {0x21f5, 0x22ff, 1}, - {0x2308, 0x230b, 1}, - {0x2320, 0x2321, 1}, - {0x237c, 0x239b, 31}, - {0x239c, 0x23b3, 1}, - {0x23dc, 0x23e1, 1}, - {0x25b7, 0x25c1, 10}, - {0x25f8, 0x25ff, 1}, - {0x266f, 0x27c0, 337}, - {0x27c1, 0x27c4, 1}, - {0x27c7, 0x27ca, 1}, - {0x27cc, 0x27ce, 2}, - {0x27cf, 0x27e5, 1}, - {0x27f0, 0x27ff, 1}, - {0x2900, 0x2982, 1}, - {0x2999, 0x29d7, 1}, - {0x29dc, 0x29fb, 1}, - {0x29fe, 0x2aff, 1}, - {0x2b30, 0x2b44, 1}, - {0x2b47, 0x2b4c, 1}, - {0xfb29, 0xfe62, 825}, - {0xfe64, 0xfe66, 1}, - {0xff0b, 0xff1c, 17}, - {0xff1d, 0xff1e, 1}, - {0xff5c, 0xff5e, 2}, - {0xffe2, 0xffe9, 7}, - {0xffea, 0xffec, 1}, - {0x1d6c1, 0x1d6db, 26}, - {0x1d6fb, 0x1d715, 26}, - {0x1d735, 0x1d74f, 26}, - {0x1d76f, 0x1d789, 26}, - {0x1d7a9, 0x1d7c3, 26}, -} - -var _Sk = []Range{ - {0x005e, 0x0060, 2}, - {0x00a8, 0x00af, 7}, - {0x00b4, 0x00b8, 4}, - {0x02c2, 0x02c5, 1}, - {0x02d2, 0x02df, 1}, - {0x02e5, 0x02eb, 1}, - {0x02ed, 0x02ef, 2}, - {0x02f0, 0x02ff, 1}, - {0x0375, 0x0384, 15}, - {0x0385, 0x1fbd, 7224}, - {0x1fbf, 0x1fc1, 1}, - {0x1fcd, 0x1fcf, 1}, - {0x1fdd, 0x1fdf, 1}, - {0x1fed, 0x1fef, 1}, - {0x1ffd, 0x1ffe, 1}, - {0x309b, 0x309c, 1}, - {0xa700, 0xa716, 1}, - {0xa720, 0xa721, 1}, - {0xa789, 0xa78a, 1}, - {0xfbb2, 0xfbc1, 1}, - {0xff3e, 0xff40, 2}, - {0xffe3, 0xffe3, 1}, -} - -var _Sc = []Range{ - {0x0024, 0x00a2, 126}, - {0x00a3, 0x00a5, 1}, - {0x060b, 0x09f2, 999}, - {0x09f3, 0x09fb, 8}, - {0x0af1, 0x0bf9, 264}, - {0x0e3f, 0x17db, 2460}, - {0x20a0, 0x20b9, 1}, - {0xa838, 0xfdfc, 21956}, - {0xfe69, 0xff04, 155}, - {0xffe0, 0xffe1, 1}, - {0xffe5, 0xffe6, 1}, -} - -var _Lu = []Range{ - {0x0041, 0x005a, 1}, - {0x00c0, 0x00d6, 1}, - {0x00d8, 0x00de, 1}, - {0x0100, 0x0136, 2}, - {0x0139, 0x0147, 2}, - {0x014a, 0x0178, 2}, - {0x0179, 0x017d, 2}, - {0x0181, 0x0182, 1}, - {0x0184, 0x0186, 2}, - {0x0187, 0x0189, 2}, - {0x018a, 0x018b, 1}, - {0x018e, 0x0191, 1}, - {0x0193, 0x0194, 1}, - {0x0196, 0x0198, 1}, - {0x019c, 0x019d, 1}, - {0x019f, 0x01a0, 1}, - {0x01a2, 0x01a6, 2}, - {0x01a7, 0x01a9, 2}, - {0x01ac, 0x01ae, 2}, - {0x01af, 0x01b1, 2}, - {0x01b2, 0x01b3, 1}, - {0x01b5, 0x01b7, 2}, - {0x01b8, 0x01bc, 4}, - {0x01c4, 0x01cd, 3}, - {0x01cf, 0x01db, 2}, - {0x01de, 0x01ee, 2}, - {0x01f1, 0x01f4, 3}, - {0x01f6, 0x01f8, 1}, - {0x01fa, 0x0232, 2}, - {0x023a, 0x023b, 1}, - {0x023d, 0x023e, 1}, - {0x0241, 0x0243, 2}, - {0x0244, 0x0246, 1}, - {0x0248, 0x024e, 2}, - {0x0370, 0x0372, 2}, - {0x0376, 0x0386, 16}, - {0x0388, 0x038a, 1}, - {0x038c, 0x038e, 2}, - {0x038f, 0x0391, 2}, - {0x0392, 0x03a1, 1}, - {0x03a3, 0x03ab, 1}, - {0x03cf, 0x03d2, 3}, - {0x03d3, 0x03d4, 1}, - {0x03d8, 0x03ee, 2}, - {0x03f4, 0x03f7, 3}, - {0x03f9, 0x03fa, 1}, - {0x03fd, 0x042f, 1}, - {0x0460, 0x0480, 2}, - {0x048a, 0x04c0, 2}, - {0x04c1, 0x04cd, 2}, - {0x04d0, 0x0526, 2}, - {0x0531, 0x0556, 1}, - {0x10a0, 0x10c5, 1}, - {0x1e00, 0x1e94, 2}, - {0x1e9e, 0x1efe, 2}, - {0x1f08, 0x1f0f, 1}, - {0x1f18, 0x1f1d, 1}, - {0x1f28, 0x1f2f, 1}, - {0x1f38, 0x1f3f, 1}, - {0x1f48, 0x1f4d, 1}, - {0x1f59, 0x1f5f, 2}, - {0x1f68, 0x1f6f, 1}, - {0x1fb8, 0x1fbb, 1}, - {0x1fc8, 0x1fcb, 1}, - {0x1fd8, 0x1fdb, 1}, - {0x1fe8, 0x1fec, 1}, - {0x1ff8, 0x1ffb, 1}, - {0x2102, 0x2107, 5}, - {0x210b, 0x210d, 1}, - {0x2110, 0x2112, 1}, - {0x2115, 0x2119, 4}, - {0x211a, 0x211d, 1}, - {0x2124, 0x212a, 2}, - {0x212b, 0x212d, 1}, - {0x2130, 0x2133, 1}, - {0x213e, 0x213f, 1}, - {0x2145, 0x2183, 62}, - {0x2c00, 0x2c2e, 1}, - {0x2c60, 0x2c62, 2}, - {0x2c63, 0x2c64, 1}, - {0x2c67, 0x2c6d, 2}, - {0x2c6e, 0x2c70, 1}, - {0x2c72, 0x2c75, 3}, - {0x2c7e, 0x2c80, 1}, - {0x2c82, 0x2ce2, 2}, - {0x2ceb, 0x2ced, 2}, - {0xa640, 0xa66c, 2}, - {0xa680, 0xa696, 2}, - {0xa722, 0xa72e, 2}, - {0xa732, 0xa76e, 2}, - {0xa779, 0xa77d, 2}, - {0xa77e, 0xa786, 2}, - {0xa78b, 0xa78d, 2}, - {0xa790, 0xa7a0, 16}, - {0xa7a2, 0xa7a8, 2}, - {0xff21, 0xff3a, 1}, - {0x10400, 0x10427, 1}, - {0x1d400, 0x1d419, 1}, - {0x1d434, 0x1d44d, 1}, - {0x1d468, 0x1d481, 1}, - {0x1d49c, 0x1d49e, 2}, - {0x1d49f, 0x1d4a5, 3}, - {0x1d4a6, 0x1d4a9, 3}, - {0x1d4aa, 0x1d4ac, 1}, - {0x1d4ae, 0x1d4b5, 1}, - {0x1d4d0, 0x1d4e9, 1}, - {0x1d504, 0x1d505, 1}, - {0x1d507, 0x1d50a, 1}, - {0x1d50d, 0x1d514, 1}, - {0x1d516, 0x1d51c, 1}, - {0x1d538, 0x1d539, 1}, - {0x1d53b, 0x1d53e, 1}, - {0x1d540, 0x1d544, 1}, - {0x1d546, 0x1d54a, 4}, - {0x1d54b, 0x1d550, 1}, - {0x1d56c, 0x1d585, 1}, - {0x1d5a0, 0x1d5b9, 1}, - {0x1d5d4, 0x1d5ed, 1}, - {0x1d608, 0x1d621, 1}, - {0x1d63c, 0x1d655, 1}, - {0x1d670, 0x1d689, 1}, - {0x1d6a8, 0x1d6c0, 1}, - {0x1d6e2, 0x1d6fa, 1}, - {0x1d71c, 0x1d734, 1}, - {0x1d756, 0x1d76e, 1}, - {0x1d790, 0x1d7a8, 1}, - {0x1d7ca, 0x1d7ca, 1}, -} - -var _Lt = []Range{ - {0x01c5, 0x01cb, 3}, - {0x01f2, 0x1f88, 7574}, - {0x1f89, 0x1f8f, 1}, - {0x1f98, 0x1f9f, 1}, - {0x1fa8, 0x1faf, 1}, - {0x1fbc, 0x1fcc, 16}, - {0x1ffc, 0x1ffc, 1}, -} - -var _Lo = []Range{ - {0x01bb, 0x01c0, 5}, - {0x01c1, 0x01c3, 1}, - {0x0294, 0x05d0, 828}, - {0x05d1, 0x05ea, 1}, - {0x05f0, 0x05f2, 1}, - {0x0620, 0x063f, 1}, - {0x0641, 0x064a, 1}, - {0x066e, 0x066f, 1}, - {0x0671, 0x06d3, 1}, - {0x06d5, 0x06ee, 25}, - {0x06ef, 0x06fa, 11}, - {0x06fb, 0x06fc, 1}, - {0x06ff, 0x0710, 17}, - {0x0712, 0x072f, 1}, - {0x074d, 0x07a5, 1}, - {0x07b1, 0x07ca, 25}, - {0x07cb, 0x07ea, 1}, - {0x0800, 0x0815, 1}, - {0x0840, 0x0858, 1}, - {0x0904, 0x0939, 1}, - {0x093d, 0x0950, 19}, - {0x0958, 0x0961, 1}, - {0x0972, 0x0977, 1}, - {0x0979, 0x097f, 1}, - {0x0985, 0x098c, 1}, - {0x098f, 0x0990, 1}, - {0x0993, 0x09a8, 1}, - {0x09aa, 0x09b0, 1}, - {0x09b2, 0x09b6, 4}, - {0x09b7, 0x09b9, 1}, - {0x09bd, 0x09ce, 17}, - {0x09dc, 0x09dd, 1}, - {0x09df, 0x09e1, 1}, - {0x09f0, 0x09f1, 1}, - {0x0a05, 0x0a0a, 1}, - {0x0a0f, 0x0a10, 1}, - {0x0a13, 0x0a28, 1}, - {0x0a2a, 0x0a30, 1}, - {0x0a32, 0x0a33, 1}, - {0x0a35, 0x0a36, 1}, - {0x0a38, 0x0a39, 1}, - {0x0a59, 0x0a5c, 1}, - {0x0a5e, 0x0a72, 20}, - {0x0a73, 0x0a74, 1}, - {0x0a85, 0x0a8d, 1}, - {0x0a8f, 0x0a91, 1}, - {0x0a93, 0x0aa8, 1}, - {0x0aaa, 0x0ab0, 1}, - {0x0ab2, 0x0ab3, 1}, - {0x0ab5, 0x0ab9, 1}, - {0x0abd, 0x0ad0, 19}, - {0x0ae0, 0x0ae1, 1}, - {0x0b05, 0x0b0c, 1}, - {0x0b0f, 0x0b10, 1}, - {0x0b13, 0x0b28, 1}, - {0x0b2a, 0x0b30, 1}, - {0x0b32, 0x0b33, 1}, - {0x0b35, 0x0b39, 1}, - {0x0b3d, 0x0b5c, 31}, - {0x0b5d, 0x0b5f, 2}, - {0x0b60, 0x0b61, 1}, - {0x0b71, 0x0b83, 18}, - {0x0b85, 0x0b8a, 1}, - {0x0b8e, 0x0b90, 1}, - {0x0b92, 0x0b95, 1}, - {0x0b99, 0x0b9a, 1}, - {0x0b9c, 0x0b9e, 2}, - {0x0b9f, 0x0ba3, 4}, - {0x0ba4, 0x0ba8, 4}, - {0x0ba9, 0x0baa, 1}, - {0x0bae, 0x0bb9, 1}, - {0x0bd0, 0x0c05, 53}, - {0x0c06, 0x0c0c, 1}, - {0x0c0e, 0x0c10, 1}, - {0x0c12, 0x0c28, 1}, - {0x0c2a, 0x0c33, 1}, - {0x0c35, 0x0c39, 1}, - {0x0c3d, 0x0c58, 27}, - {0x0c59, 0x0c60, 7}, - {0x0c61, 0x0c85, 36}, - {0x0c86, 0x0c8c, 1}, - {0x0c8e, 0x0c90, 1}, - {0x0c92, 0x0ca8, 1}, - {0x0caa, 0x0cb3, 1}, - {0x0cb5, 0x0cb9, 1}, - {0x0cbd, 0x0cde, 33}, - {0x0ce0, 0x0ce1, 1}, - {0x0cf1, 0x0cf2, 1}, - {0x0d05, 0x0d0c, 1}, - {0x0d0e, 0x0d10, 1}, - {0x0d12, 0x0d3a, 1}, - {0x0d3d, 0x0d4e, 17}, - {0x0d60, 0x0d61, 1}, - {0x0d7a, 0x0d7f, 1}, - {0x0d85, 0x0d96, 1}, - {0x0d9a, 0x0db1, 1}, - {0x0db3, 0x0dbb, 1}, - {0x0dbd, 0x0dc0, 3}, - {0x0dc1, 0x0dc6, 1}, - {0x0e01, 0x0e30, 1}, - {0x0e32, 0x0e33, 1}, - {0x0e40, 0x0e45, 1}, - {0x0e81, 0x0e82, 1}, - {0x0e84, 0x0e87, 3}, - {0x0e88, 0x0e8a, 2}, - {0x0e8d, 0x0e94, 7}, - {0x0e95, 0x0e97, 1}, - {0x0e99, 0x0e9f, 1}, - {0x0ea1, 0x0ea3, 1}, - {0x0ea5, 0x0ea7, 2}, - {0x0eaa, 0x0eab, 1}, - {0x0ead, 0x0eb0, 1}, - {0x0eb2, 0x0eb3, 1}, - {0x0ebd, 0x0ec0, 3}, - {0x0ec1, 0x0ec4, 1}, - {0x0edc, 0x0edd, 1}, - {0x0f00, 0x0f40, 64}, - {0x0f41, 0x0f47, 1}, - {0x0f49, 0x0f6c, 1}, - {0x0f88, 0x0f8c, 1}, - {0x1000, 0x102a, 1}, - {0x103f, 0x1050, 17}, - {0x1051, 0x1055, 1}, - {0x105a, 0x105d, 1}, - {0x1061, 0x1065, 4}, - {0x1066, 0x106e, 8}, - {0x106f, 0x1070, 1}, - {0x1075, 0x1081, 1}, - {0x108e, 0x10d0, 66}, - {0x10d1, 0x10fa, 1}, - {0x1100, 0x1248, 1}, - {0x124a, 0x124d, 1}, - {0x1250, 0x1256, 1}, - {0x1258, 0x125a, 2}, - {0x125b, 0x125d, 1}, - {0x1260, 0x1288, 1}, - {0x128a, 0x128d, 1}, - {0x1290, 0x12b0, 1}, - {0x12b2, 0x12b5, 1}, - {0x12b8, 0x12be, 1}, - {0x12c0, 0x12c2, 2}, - {0x12c3, 0x12c5, 1}, - {0x12c8, 0x12d6, 1}, - {0x12d8, 0x1310, 1}, - {0x1312, 0x1315, 1}, - {0x1318, 0x135a, 1}, - {0x1380, 0x138f, 1}, - {0x13a0, 0x13f4, 1}, - {0x1401, 0x166c, 1}, - {0x166f, 0x167f, 1}, - {0x1681, 0x169a, 1}, - {0x16a0, 0x16ea, 1}, - {0x1700, 0x170c, 1}, - {0x170e, 0x1711, 1}, - {0x1720, 0x1731, 1}, - {0x1740, 0x1751, 1}, - {0x1760, 0x176c, 1}, - {0x176e, 0x1770, 1}, - {0x1780, 0x17b3, 1}, - {0x17dc, 0x1820, 68}, - {0x1821, 0x1842, 1}, - {0x1844, 0x1877, 1}, - {0x1880, 0x18a8, 1}, - {0x18aa, 0x18b0, 6}, - {0x18b1, 0x18f5, 1}, - {0x1900, 0x191c, 1}, - {0x1950, 0x196d, 1}, - {0x1970, 0x1974, 1}, - {0x1980, 0x19ab, 1}, - {0x19c1, 0x19c7, 1}, - {0x1a00, 0x1a16, 1}, - {0x1a20, 0x1a54, 1}, - {0x1b05, 0x1b33, 1}, - {0x1b45, 0x1b4b, 1}, - {0x1b83, 0x1ba0, 1}, - {0x1bae, 0x1baf, 1}, - {0x1bc0, 0x1be5, 1}, - {0x1c00, 0x1c23, 1}, - {0x1c4d, 0x1c4f, 1}, - {0x1c5a, 0x1c77, 1}, - {0x1ce9, 0x1cec, 1}, - {0x1cee, 0x1cf1, 1}, - {0x2135, 0x2138, 1}, - {0x2d30, 0x2d65, 1}, - {0x2d80, 0x2d96, 1}, - {0x2da0, 0x2da6, 1}, - {0x2da8, 0x2dae, 1}, - {0x2db0, 0x2db6, 1}, - {0x2db8, 0x2dbe, 1}, - {0x2dc0, 0x2dc6, 1}, - {0x2dc8, 0x2dce, 1}, - {0x2dd0, 0x2dd6, 1}, - {0x2dd8, 0x2dde, 1}, - {0x3006, 0x303c, 54}, - {0x3041, 0x3096, 1}, - {0x309f, 0x30a1, 2}, - {0x30a2, 0x30fa, 1}, - {0x30ff, 0x3105, 6}, - {0x3106, 0x312d, 1}, - {0x3131, 0x318e, 1}, - {0x31a0, 0x31ba, 1}, - {0x31f0, 0x31ff, 1}, - {0x3400, 0x4db5, 1}, - {0x4e00, 0x9fcb, 1}, - {0xa000, 0xa014, 1}, - {0xa016, 0xa48c, 1}, - {0xa4d0, 0xa4f7, 1}, - {0xa500, 0xa60b, 1}, - {0xa610, 0xa61f, 1}, - {0xa62a, 0xa62b, 1}, - {0xa66e, 0xa6a0, 50}, - {0xa6a1, 0xa6e5, 1}, - {0xa7fb, 0xa801, 1}, - {0xa803, 0xa805, 1}, - {0xa807, 0xa80a, 1}, - {0xa80c, 0xa822, 1}, - {0xa840, 0xa873, 1}, - {0xa882, 0xa8b3, 1}, - {0xa8f2, 0xa8f7, 1}, - {0xa8fb, 0xa90a, 15}, - {0xa90b, 0xa925, 1}, - {0xa930, 0xa946, 1}, - {0xa960, 0xa97c, 1}, - {0xa984, 0xa9b2, 1}, - {0xaa00, 0xaa28, 1}, - {0xaa40, 0xaa42, 1}, - {0xaa44, 0xaa4b, 1}, - {0xaa60, 0xaa6f, 1}, - {0xaa71, 0xaa76, 1}, - {0xaa7a, 0xaa80, 6}, - {0xaa81, 0xaaaf, 1}, - {0xaab1, 0xaab5, 4}, - {0xaab6, 0xaab9, 3}, - {0xaaba, 0xaabd, 1}, - {0xaac0, 0xaac2, 2}, - {0xaadb, 0xaadc, 1}, - {0xab01, 0xab06, 1}, - {0xab09, 0xab0e, 1}, - {0xab11, 0xab16, 1}, - {0xab20, 0xab26, 1}, - {0xab28, 0xab2e, 1}, - {0xabc0, 0xabe2, 1}, - {0xac00, 0xd7a3, 1}, - {0xd7b0, 0xd7c6, 1}, - {0xd7cb, 0xd7fb, 1}, - {0xf900, 0xfa2d, 1}, - {0xfa30, 0xfa6d, 1}, - {0xfa70, 0xfad9, 1}, - {0xfb1d, 0xfb1f, 2}, - {0xfb20, 0xfb28, 1}, - {0xfb2a, 0xfb36, 1}, - {0xfb38, 0xfb3c, 1}, - {0xfb3e, 0xfb40, 2}, - {0xfb41, 0xfb43, 2}, - {0xfb44, 0xfb46, 2}, - {0xfb47, 0xfbb1, 1}, - {0xfbd3, 0xfd3d, 1}, - {0xfd50, 0xfd8f, 1}, - {0xfd92, 0xfdc7, 1}, - {0xfdf0, 0xfdfb, 1}, - {0xfe70, 0xfe74, 1}, - {0xfe76, 0xfefc, 1}, - {0xff66, 0xff6f, 1}, - {0xff71, 0xff9d, 1}, - {0xffa0, 0xffbe, 1}, - {0xffc2, 0xffc7, 1}, - {0xffca, 0xffcf, 1}, - {0xffd2, 0xffd7, 1}, - {0xffda, 0xffdc, 1}, - {0x10000, 0x1000b, 1}, - {0x1000d, 0x10026, 1}, - {0x10028, 0x1003a, 1}, - {0x1003c, 0x1003d, 1}, - {0x1003f, 0x1004d, 1}, - {0x10050, 0x1005d, 1}, - {0x10080, 0x100fa, 1}, - {0x10280, 0x1029c, 1}, - {0x102a0, 0x102d0, 1}, - {0x10300, 0x1031e, 1}, - {0x10330, 0x10340, 1}, - {0x10342, 0x10349, 1}, - {0x10380, 0x1039d, 1}, - {0x103a0, 0x103c3, 1}, - {0x103c8, 0x103cf, 1}, - {0x10450, 0x1049d, 1}, - {0x10800, 0x10805, 1}, - {0x10808, 0x1080a, 2}, - {0x1080b, 0x10835, 1}, - {0x10837, 0x10838, 1}, - {0x1083c, 0x1083f, 3}, - {0x10840, 0x10855, 1}, - {0x10900, 0x10915, 1}, - {0x10920, 0x10939, 1}, - {0x10a00, 0x10a10, 16}, - {0x10a11, 0x10a13, 1}, - {0x10a15, 0x10a17, 1}, - {0x10a19, 0x10a33, 1}, - {0x10a60, 0x10a7c, 1}, - {0x10b00, 0x10b35, 1}, - {0x10b40, 0x10b55, 1}, - {0x10b60, 0x10b72, 1}, - {0x10c00, 0x10c48, 1}, - {0x11003, 0x11037, 1}, - {0x11083, 0x110af, 1}, - {0x12000, 0x1236e, 1}, - {0x13000, 0x1342e, 1}, - {0x16800, 0x16a38, 1}, - {0x1b000, 0x1b001, 1}, - {0x20000, 0x2a6d6, 1}, - {0x2a700, 0x2b734, 1}, - {0x2b740, 0x2b81d, 1}, - {0x2f800, 0x2fa1d, 1}, +var Categories = map[string]*RangeTable{ + "Lm": Lm, + "Ll": Ll, + "C": C, + "M": M, + "L": L, + "N": N, + "P": P, + "S": S, + "Z": Z, + "Me": Me, + "Mc": Mc, + "Mn": Mn, + "Zl": Zl, + "Zp": Zp, + "Zs": Zs, + "Cs": Cs, + "Co": Co, + "Cf": Cf, + "Cc": Cc, + "Po": Po, + "Pi": Pi, + "Pf": Pf, + "Pe": Pe, + "Pd": Pd, + "Pc": Pc, + "Ps": Ps, + "Nd": Nd, + "Nl": Nl, + "No": No, + "So": So, + "Sm": Sm, + "Sk": Sk, + "Sc": Sc, + "Lu": Lu, + "Lt": Lt, + "Lo": Lo, +} + +var _Lm = &RangeTable{ + R16: []Range16{ + {0x02b0, 0x02c1, 1}, + {0x02c6, 0x02d1, 1}, + {0x02e0, 0x02e4, 1}, + {0x02ec, 0x02ee, 2}, + {0x0374, 0x037a, 6}, + {0x0559, 0x0640, 231}, + {0x06e5, 0x06e6, 1}, + {0x07f4, 0x07f5, 1}, + {0x07fa, 0x081a, 32}, + {0x0824, 0x0828, 4}, + {0x0971, 0x0e46, 1237}, + {0x0ec6, 0x10fc, 566}, + {0x17d7, 0x1843, 108}, + {0x1aa7, 0x1c78, 465}, + {0x1c79, 0x1c7d, 1}, + {0x1d2c, 0x1d61, 1}, + {0x1d78, 0x1d9b, 35}, + {0x1d9c, 0x1dbf, 1}, + {0x2071, 0x207f, 14}, + {0x2090, 0x209c, 1}, + {0x2c7d, 0x2d6f, 242}, + {0x2e2f, 0x3005, 470}, + {0x3031, 0x3035, 1}, + {0x303b, 0x309d, 98}, + {0x309e, 0x30fc, 94}, + {0x30fd, 0x30fe, 1}, + {0xa015, 0xa4f8, 1251}, + {0xa4f9, 0xa4fd, 1}, + {0xa60c, 0xa67f, 115}, + {0xa717, 0xa71f, 1}, + {0xa770, 0xa788, 24}, + {0xa9cf, 0xaa70, 161}, + {0xaadd, 0xff70, 21651}, + {0xff9e, 0xff9f, 1}, + }, +} + +var _Ll = &RangeTable{ + R16: []Range16{ + {0x0061, 0x007a, 1}, + {0x00aa, 0x00b5, 11}, + {0x00ba, 0x00df, 37}, + {0x00e0, 0x00f6, 1}, + {0x00f8, 0x00ff, 1}, + {0x0101, 0x0137, 2}, + {0x0138, 0x0148, 2}, + {0x0149, 0x0177, 2}, + {0x017a, 0x017e, 2}, + {0x017f, 0x0180, 1}, + {0x0183, 0x0185, 2}, + {0x0188, 0x018c, 4}, + {0x018d, 0x0192, 5}, + {0x0195, 0x0199, 4}, + {0x019a, 0x019b, 1}, + {0x019e, 0x01a1, 3}, + {0x01a3, 0x01a5, 2}, + {0x01a8, 0x01aa, 2}, + {0x01ab, 0x01ad, 2}, + {0x01b0, 0x01b4, 4}, + {0x01b6, 0x01b9, 3}, + {0x01ba, 0x01bd, 3}, + {0x01be, 0x01bf, 1}, + {0x01c6, 0x01cc, 3}, + {0x01ce, 0x01dc, 2}, + {0x01dd, 0x01ef, 2}, + {0x01f0, 0x01f3, 3}, + {0x01f5, 0x01f9, 4}, + {0x01fb, 0x0233, 2}, + {0x0234, 0x0239, 1}, + {0x023c, 0x023f, 3}, + {0x0240, 0x0242, 2}, + {0x0247, 0x024f, 2}, + {0x0250, 0x0293, 1}, + {0x0295, 0x02af, 1}, + {0x0371, 0x0373, 2}, + {0x0377, 0x037b, 4}, + {0x037c, 0x037d, 1}, + {0x0390, 0x03ac, 28}, + {0x03ad, 0x03ce, 1}, + {0x03d0, 0x03d1, 1}, + {0x03d5, 0x03d7, 1}, + {0x03d9, 0x03ef, 2}, + {0x03f0, 0x03f3, 1}, + {0x03f5, 0x03fb, 3}, + {0x03fc, 0x0430, 52}, + {0x0431, 0x045f, 1}, + {0x0461, 0x0481, 2}, + {0x048b, 0x04bf, 2}, + {0x04c2, 0x04ce, 2}, + {0x04cf, 0x0527, 2}, + {0x0561, 0x0587, 1}, + {0x1d00, 0x1d2b, 1}, + {0x1d62, 0x1d77, 1}, + {0x1d79, 0x1d9a, 1}, + {0x1e01, 0x1e95, 2}, + {0x1e96, 0x1e9d, 1}, + {0x1e9f, 0x1eff, 2}, + {0x1f00, 0x1f07, 1}, + {0x1f10, 0x1f15, 1}, + {0x1f20, 0x1f27, 1}, + {0x1f30, 0x1f37, 1}, + {0x1f40, 0x1f45, 1}, + {0x1f50, 0x1f57, 1}, + {0x1f60, 0x1f67, 1}, + {0x1f70, 0x1f7d, 1}, + {0x1f80, 0x1f87, 1}, + {0x1f90, 0x1f97, 1}, + {0x1fa0, 0x1fa7, 1}, + {0x1fb0, 0x1fb4, 1}, + {0x1fb6, 0x1fb7, 1}, + {0x1fbe, 0x1fc2, 4}, + {0x1fc3, 0x1fc4, 1}, + {0x1fc6, 0x1fc7, 1}, + {0x1fd0, 0x1fd3, 1}, + {0x1fd6, 0x1fd7, 1}, + {0x1fe0, 0x1fe7, 1}, + {0x1ff2, 0x1ff4, 1}, + {0x1ff6, 0x1ff7, 1}, + {0x210a, 0x210e, 4}, + {0x210f, 0x2113, 4}, + {0x212f, 0x2139, 5}, + {0x213c, 0x213d, 1}, + {0x2146, 0x2149, 1}, + {0x214e, 0x2184, 54}, + {0x2c30, 0x2c5e, 1}, + {0x2c61, 0x2c65, 4}, + {0x2c66, 0x2c6c, 2}, + {0x2c71, 0x2c73, 2}, + {0x2c74, 0x2c76, 2}, + {0x2c77, 0x2c7c, 1}, + {0x2c81, 0x2ce3, 2}, + {0x2ce4, 0x2cec, 8}, + {0x2cee, 0x2d00, 18}, + {0x2d01, 0x2d25, 1}, + {0xa641, 0xa66d, 2}, + {0xa681, 0xa697, 2}, + {0xa723, 0xa72f, 2}, + {0xa730, 0xa731, 1}, + {0xa733, 0xa771, 2}, + {0xa772, 0xa778, 1}, + {0xa77a, 0xa77c, 2}, + {0xa77f, 0xa787, 2}, + {0xa78c, 0xa78e, 2}, + {0xa791, 0xa7a1, 16}, + {0xa7a3, 0xa7a9, 2}, + {0xa7fa, 0xfb00, 21254}, + {0xfb01, 0xfb06, 1}, + {0xfb13, 0xfb17, 1}, + {0xff41, 0xff5a, 1}, + }, + R32: []Range32{ + {0x10428, 0x1044f, 1}, + {0x1d41a, 0x1d433, 1}, + {0x1d44e, 0x1d454, 1}, + {0x1d456, 0x1d467, 1}, + {0x1d482, 0x1d49b, 1}, + {0x1d4b6, 0x1d4b9, 1}, + {0x1d4bb, 0x1d4bd, 2}, + {0x1d4be, 0x1d4c3, 1}, + {0x1d4c5, 0x1d4cf, 1}, + {0x1d4ea, 0x1d503, 1}, + {0x1d51e, 0x1d537, 1}, + {0x1d552, 0x1d56b, 1}, + {0x1d586, 0x1d59f, 1}, + {0x1d5ba, 0x1d5d3, 1}, + {0x1d5ee, 0x1d607, 1}, + {0x1d622, 0x1d63b, 1}, + {0x1d656, 0x1d66f, 1}, + {0x1d68a, 0x1d6a5, 1}, + {0x1d6c2, 0x1d6da, 1}, + {0x1d6dc, 0x1d6e1, 1}, + {0x1d6fc, 0x1d714, 1}, + {0x1d716, 0x1d71b, 1}, + {0x1d736, 0x1d74e, 1}, + {0x1d750, 0x1d755, 1}, + {0x1d770, 0x1d788, 1}, + {0x1d78a, 0x1d78f, 1}, + {0x1d7aa, 0x1d7c2, 1}, + {0x1d7c4, 0x1d7c9, 1}, + {0x1d7cb, 0x1d7cb, 1}, + }, +} + +var _C = &RangeTable{ + R16: []Range16{ + {0x0001, 0x001f, 1}, + {0x007f, 0x009f, 1}, + {0x00ad, 0x0600, 1363}, + {0x0601, 0x0603, 1}, + {0x06dd, 0x070f, 50}, + {0x17b4, 0x17b5, 1}, + {0x200b, 0x200f, 1}, + {0x202a, 0x202e, 1}, + {0x2060, 0x2064, 1}, + {0x206a, 0x206f, 1}, + {0xd800, 0xf8ff, 1}, + {0xfeff, 0xfff9, 250}, + {0xfffa, 0xfffb, 1}, + }, + R32: []Range32{ + {0x110bd, 0x1d173, 49334}, + {0x1d174, 0x1d17a, 1}, + {0xe0001, 0xe0020, 31}, + {0xe0021, 0xe007f, 1}, + {0xf0000, 0xffffd, 1}, + {0x100000, 0x10fffd, 1}, + }, +} + +var _M = &RangeTable{ + R16: []Range16{ + {0x0300, 0x036f, 1}, + {0x0483, 0x0489, 1}, + {0x0591, 0x05bd, 1}, + {0x05bf, 0x05c1, 2}, + {0x05c2, 0x05c4, 2}, + {0x05c5, 0x05c7, 2}, + {0x0610, 0x061a, 1}, + {0x064b, 0x065f, 1}, + {0x0670, 0x06d6, 102}, + {0x06d7, 0x06dc, 1}, + {0x06df, 0x06e4, 1}, + {0x06e7, 0x06e8, 1}, + {0x06ea, 0x06ed, 1}, + {0x0711, 0x0730, 31}, + {0x0731, 0x074a, 1}, + {0x07a6, 0x07b0, 1}, + {0x07eb, 0x07f3, 1}, + {0x0816, 0x0819, 1}, + {0x081b, 0x0823, 1}, + {0x0825, 0x0827, 1}, + {0x0829, 0x082d, 1}, + {0x0859, 0x085b, 1}, + {0x0900, 0x0903, 1}, + {0x093a, 0x093c, 1}, + {0x093e, 0x094f, 1}, + {0x0951, 0x0957, 1}, + {0x0962, 0x0963, 1}, + {0x0981, 0x0983, 1}, + {0x09bc, 0x09be, 2}, + {0x09bf, 0x09c4, 1}, + {0x09c7, 0x09c8, 1}, + {0x09cb, 0x09cd, 1}, + {0x09d7, 0x09e2, 11}, + {0x09e3, 0x0a01, 30}, + {0x0a02, 0x0a03, 1}, + {0x0a3c, 0x0a3e, 2}, + {0x0a3f, 0x0a42, 1}, + {0x0a47, 0x0a48, 1}, + {0x0a4b, 0x0a4d, 1}, + {0x0a51, 0x0a70, 31}, + {0x0a71, 0x0a75, 4}, + {0x0a81, 0x0a83, 1}, + {0x0abc, 0x0abe, 2}, + {0x0abf, 0x0ac5, 1}, + {0x0ac7, 0x0ac9, 1}, + {0x0acb, 0x0acd, 1}, + {0x0ae2, 0x0ae3, 1}, + {0x0b01, 0x0b03, 1}, + {0x0b3c, 0x0b3e, 2}, + {0x0b3f, 0x0b44, 1}, + {0x0b47, 0x0b48, 1}, + {0x0b4b, 0x0b4d, 1}, + {0x0b56, 0x0b57, 1}, + {0x0b62, 0x0b63, 1}, + {0x0b82, 0x0bbe, 60}, + {0x0bbf, 0x0bc2, 1}, + {0x0bc6, 0x0bc8, 1}, + {0x0bca, 0x0bcd, 1}, + {0x0bd7, 0x0c01, 42}, + {0x0c02, 0x0c03, 1}, + {0x0c3e, 0x0c44, 1}, + {0x0c46, 0x0c48, 1}, + {0x0c4a, 0x0c4d, 1}, + {0x0c55, 0x0c56, 1}, + {0x0c62, 0x0c63, 1}, + {0x0c82, 0x0c83, 1}, + {0x0cbc, 0x0cbe, 2}, + {0x0cbf, 0x0cc4, 1}, + {0x0cc6, 0x0cc8, 1}, + {0x0cca, 0x0ccd, 1}, + {0x0cd5, 0x0cd6, 1}, + {0x0ce2, 0x0ce3, 1}, + {0x0d02, 0x0d03, 1}, + {0x0d3e, 0x0d44, 1}, + {0x0d46, 0x0d48, 1}, + {0x0d4a, 0x0d4d, 1}, + {0x0d57, 0x0d62, 11}, + {0x0d63, 0x0d82, 31}, + {0x0d83, 0x0dca, 71}, + {0x0dcf, 0x0dd4, 1}, + {0x0dd6, 0x0dd8, 2}, + {0x0dd9, 0x0ddf, 1}, + {0x0df2, 0x0df3, 1}, + {0x0e31, 0x0e34, 3}, + {0x0e35, 0x0e3a, 1}, + {0x0e47, 0x0e4e, 1}, + {0x0eb1, 0x0eb4, 3}, + {0x0eb5, 0x0eb9, 1}, + {0x0ebb, 0x0ebc, 1}, + {0x0ec8, 0x0ecd, 1}, + {0x0f18, 0x0f19, 1}, + {0x0f35, 0x0f39, 2}, + {0x0f3e, 0x0f3f, 1}, + {0x0f71, 0x0f84, 1}, + {0x0f86, 0x0f87, 1}, + {0x0f8d, 0x0f97, 1}, + {0x0f99, 0x0fbc, 1}, + {0x0fc6, 0x102b, 101}, + {0x102c, 0x103e, 1}, + {0x1056, 0x1059, 1}, + {0x105e, 0x1060, 1}, + {0x1062, 0x1064, 1}, + {0x1067, 0x106d, 1}, + {0x1071, 0x1074, 1}, + {0x1082, 0x108d, 1}, + {0x108f, 0x109a, 11}, + {0x109b, 0x109d, 1}, + {0x135d, 0x135f, 1}, + {0x1712, 0x1714, 1}, + {0x1732, 0x1734, 1}, + {0x1752, 0x1753, 1}, + {0x1772, 0x1773, 1}, + {0x17b6, 0x17d3, 1}, + {0x17dd, 0x180b, 46}, + {0x180c, 0x180d, 1}, + {0x18a9, 0x1920, 119}, + {0x1921, 0x192b, 1}, + {0x1930, 0x193b, 1}, + {0x19b0, 0x19c0, 1}, + {0x19c8, 0x19c9, 1}, + {0x1a17, 0x1a1b, 1}, + {0x1a55, 0x1a5e, 1}, + {0x1a60, 0x1a7c, 1}, + {0x1a7f, 0x1b00, 129}, + {0x1b01, 0x1b04, 1}, + {0x1b34, 0x1b44, 1}, + {0x1b6b, 0x1b73, 1}, + {0x1b80, 0x1b82, 1}, + {0x1ba1, 0x1baa, 1}, + {0x1be6, 0x1bf3, 1}, + {0x1c24, 0x1c37, 1}, + {0x1cd0, 0x1cd2, 1}, + {0x1cd4, 0x1ce8, 1}, + {0x1ced, 0x1cf2, 5}, + {0x1dc0, 0x1de6, 1}, + {0x1dfc, 0x1dff, 1}, + {0x20d0, 0x20f0, 1}, + {0x2cef, 0x2cf1, 1}, + {0x2d7f, 0x2de0, 97}, + {0x2de1, 0x2dff, 1}, + {0x302a, 0x302f, 1}, + {0x3099, 0x309a, 1}, + {0xa66f, 0xa672, 1}, + {0xa67c, 0xa67d, 1}, + {0xa6f0, 0xa6f1, 1}, + {0xa802, 0xa806, 4}, + {0xa80b, 0xa823, 24}, + {0xa824, 0xa827, 1}, + {0xa880, 0xa881, 1}, + {0xa8b4, 0xa8c4, 1}, + {0xa8e0, 0xa8f1, 1}, + {0xa926, 0xa92d, 1}, + {0xa947, 0xa953, 1}, + {0xa980, 0xa983, 1}, + {0xa9b3, 0xa9c0, 1}, + {0xaa29, 0xaa36, 1}, + {0xaa43, 0xaa4c, 9}, + {0xaa4d, 0xaa7b, 46}, + {0xaab0, 0xaab2, 2}, + {0xaab3, 0xaab4, 1}, + {0xaab7, 0xaab8, 1}, + {0xaabe, 0xaabf, 1}, + {0xaac1, 0xabe3, 290}, + {0xabe4, 0xabea, 1}, + {0xabec, 0xabed, 1}, + {0xfb1e, 0xfe00, 738}, + {0xfe01, 0xfe0f, 1}, + {0xfe20, 0xfe26, 1}, + }, + R32: []Range32{ + {0x101fd, 0x10a01, 2052}, + {0x10a02, 0x10a03, 1}, + {0x10a05, 0x10a06, 1}, + {0x10a0c, 0x10a0f, 1}, + {0x10a38, 0x10a3a, 1}, + {0x10a3f, 0x11000, 1473}, + {0x11001, 0x11002, 1}, + {0x11038, 0x11046, 1}, + {0x11080, 0x11082, 1}, + {0x110b0, 0x110ba, 1}, + {0x1d165, 0x1d169, 1}, + {0x1d16d, 0x1d172, 1}, + {0x1d17b, 0x1d182, 1}, + {0x1d185, 0x1d18b, 1}, + {0x1d1aa, 0x1d1ad, 1}, + {0x1d242, 0x1d244, 1}, + {0xe0100, 0xe01ef, 1}, + }, +} + +var _L = &RangeTable{ + R16: []Range16{ + {0x0041, 0x005a, 1}, + {0x0061, 0x007a, 1}, + {0x00aa, 0x00b5, 11}, + {0x00ba, 0x00c0, 6}, + {0x00c1, 0x00d6, 1}, + {0x00d8, 0x00f6, 1}, + {0x00f8, 0x02c1, 1}, + {0x02c6, 0x02d1, 1}, + {0x02e0, 0x02e4, 1}, + {0x02ec, 0x02ee, 2}, + {0x0370, 0x0374, 1}, + {0x0376, 0x0377, 1}, + {0x037a, 0x037d, 1}, + {0x0386, 0x0388, 2}, + {0x0389, 0x038a, 1}, + {0x038c, 0x038e, 2}, + {0x038f, 0x03a1, 1}, + {0x03a3, 0x03f5, 1}, + {0x03f7, 0x0481, 1}, + {0x048a, 0x0527, 1}, + {0x0531, 0x0556, 1}, + {0x0559, 0x0561, 8}, + {0x0562, 0x0587, 1}, + {0x05d0, 0x05ea, 1}, + {0x05f0, 0x05f2, 1}, + {0x0620, 0x064a, 1}, + {0x066e, 0x066f, 1}, + {0x0671, 0x06d3, 1}, + {0x06d5, 0x06e5, 16}, + {0x06e6, 0x06ee, 8}, + {0x06ef, 0x06fa, 11}, + {0x06fb, 0x06fc, 1}, + {0x06ff, 0x0710, 17}, + {0x0712, 0x072f, 1}, + {0x074d, 0x07a5, 1}, + {0x07b1, 0x07ca, 25}, + {0x07cb, 0x07ea, 1}, + {0x07f4, 0x07f5, 1}, + {0x07fa, 0x0800, 6}, + {0x0801, 0x0815, 1}, + {0x081a, 0x0824, 10}, + {0x0828, 0x0840, 24}, + {0x0841, 0x0858, 1}, + {0x0904, 0x0939, 1}, + {0x093d, 0x0950, 19}, + {0x0958, 0x0961, 1}, + {0x0971, 0x0977, 1}, + {0x0979, 0x097f, 1}, + {0x0985, 0x098c, 1}, + {0x098f, 0x0990, 1}, + {0x0993, 0x09a8, 1}, + {0x09aa, 0x09b0, 1}, + {0x09b2, 0x09b6, 4}, + {0x09b7, 0x09b9, 1}, + {0x09bd, 0x09ce, 17}, + {0x09dc, 0x09dd, 1}, + {0x09df, 0x09e1, 1}, + {0x09f0, 0x09f1, 1}, + {0x0a05, 0x0a0a, 1}, + {0x0a0f, 0x0a10, 1}, + {0x0a13, 0x0a28, 1}, + {0x0a2a, 0x0a30, 1}, + {0x0a32, 0x0a33, 1}, + {0x0a35, 0x0a36, 1}, + {0x0a38, 0x0a39, 1}, + {0x0a59, 0x0a5c, 1}, + {0x0a5e, 0x0a72, 20}, + {0x0a73, 0x0a74, 1}, + {0x0a85, 0x0a8d, 1}, + {0x0a8f, 0x0a91, 1}, + {0x0a93, 0x0aa8, 1}, + {0x0aaa, 0x0ab0, 1}, + {0x0ab2, 0x0ab3, 1}, + {0x0ab5, 0x0ab9, 1}, + {0x0abd, 0x0ad0, 19}, + {0x0ae0, 0x0ae1, 1}, + {0x0b05, 0x0b0c, 1}, + {0x0b0f, 0x0b10, 1}, + {0x0b13, 0x0b28, 1}, + {0x0b2a, 0x0b30, 1}, + {0x0b32, 0x0b33, 1}, + {0x0b35, 0x0b39, 1}, + {0x0b3d, 0x0b5c, 31}, + {0x0b5d, 0x0b5f, 2}, + {0x0b60, 0x0b61, 1}, + {0x0b71, 0x0b83, 18}, + {0x0b85, 0x0b8a, 1}, + {0x0b8e, 0x0b90, 1}, + {0x0b92, 0x0b95, 1}, + {0x0b99, 0x0b9a, 1}, + {0x0b9c, 0x0b9e, 2}, + {0x0b9f, 0x0ba3, 4}, + {0x0ba4, 0x0ba8, 4}, + {0x0ba9, 0x0baa, 1}, + {0x0bae, 0x0bb9, 1}, + {0x0bd0, 0x0c05, 53}, + {0x0c06, 0x0c0c, 1}, + {0x0c0e, 0x0c10, 1}, + {0x0c12, 0x0c28, 1}, + {0x0c2a, 0x0c33, 1}, + {0x0c35, 0x0c39, 1}, + {0x0c3d, 0x0c58, 27}, + {0x0c59, 0x0c60, 7}, + {0x0c61, 0x0c85, 36}, + {0x0c86, 0x0c8c, 1}, + {0x0c8e, 0x0c90, 1}, + {0x0c92, 0x0ca8, 1}, + {0x0caa, 0x0cb3, 1}, + {0x0cb5, 0x0cb9, 1}, + {0x0cbd, 0x0cde, 33}, + {0x0ce0, 0x0ce1, 1}, + {0x0cf1, 0x0cf2, 1}, + {0x0d05, 0x0d0c, 1}, + {0x0d0e, 0x0d10, 1}, + {0x0d12, 0x0d3a, 1}, + {0x0d3d, 0x0d4e, 17}, + {0x0d60, 0x0d61, 1}, + {0x0d7a, 0x0d7f, 1}, + {0x0d85, 0x0d96, 1}, + {0x0d9a, 0x0db1, 1}, + {0x0db3, 0x0dbb, 1}, + {0x0dbd, 0x0dc0, 3}, + {0x0dc1, 0x0dc6, 1}, + {0x0e01, 0x0e30, 1}, + {0x0e32, 0x0e33, 1}, + {0x0e40, 0x0e46, 1}, + {0x0e81, 0x0e82, 1}, + {0x0e84, 0x0e87, 3}, + {0x0e88, 0x0e8a, 2}, + {0x0e8d, 0x0e94, 7}, + {0x0e95, 0x0e97, 1}, + {0x0e99, 0x0e9f, 1}, + {0x0ea1, 0x0ea3, 1}, + {0x0ea5, 0x0ea7, 2}, + {0x0eaa, 0x0eab, 1}, + {0x0ead, 0x0eb0, 1}, + {0x0eb2, 0x0eb3, 1}, + {0x0ebd, 0x0ec0, 3}, + {0x0ec1, 0x0ec4, 1}, + {0x0ec6, 0x0edc, 22}, + {0x0edd, 0x0f00, 35}, + {0x0f40, 0x0f47, 1}, + {0x0f49, 0x0f6c, 1}, + {0x0f88, 0x0f8c, 1}, + {0x1000, 0x102a, 1}, + {0x103f, 0x1050, 17}, + {0x1051, 0x1055, 1}, + {0x105a, 0x105d, 1}, + {0x1061, 0x1065, 4}, + {0x1066, 0x106e, 8}, + {0x106f, 0x1070, 1}, + {0x1075, 0x1081, 1}, + {0x108e, 0x10a0, 18}, + {0x10a1, 0x10c5, 1}, + {0x10d0, 0x10fa, 1}, + {0x10fc, 0x1100, 4}, + {0x1101, 0x1248, 1}, + {0x124a, 0x124d, 1}, + {0x1250, 0x1256, 1}, + {0x1258, 0x125a, 2}, + {0x125b, 0x125d, 1}, + {0x1260, 0x1288, 1}, + {0x128a, 0x128d, 1}, + {0x1290, 0x12b0, 1}, + {0x12b2, 0x12b5, 1}, + {0x12b8, 0x12be, 1}, + {0x12c0, 0x12c2, 2}, + {0x12c3, 0x12c5, 1}, + {0x12c8, 0x12d6, 1}, + {0x12d8, 0x1310, 1}, + {0x1312, 0x1315, 1}, + {0x1318, 0x135a, 1}, + {0x1380, 0x138f, 1}, + {0x13a0, 0x13f4, 1}, + {0x1401, 0x166c, 1}, + {0x166f, 0x167f, 1}, + {0x1681, 0x169a, 1}, + {0x16a0, 0x16ea, 1}, + {0x1700, 0x170c, 1}, + {0x170e, 0x1711, 1}, + {0x1720, 0x1731, 1}, + {0x1740, 0x1751, 1}, + {0x1760, 0x176c, 1}, + {0x176e, 0x1770, 1}, + {0x1780, 0x17b3, 1}, + {0x17d7, 0x17dc, 5}, + {0x1820, 0x1877, 1}, + {0x1880, 0x18a8, 1}, + {0x18aa, 0x18b0, 6}, + {0x18b1, 0x18f5, 1}, + {0x1900, 0x191c, 1}, + {0x1950, 0x196d, 1}, + {0x1970, 0x1974, 1}, + {0x1980, 0x19ab, 1}, + {0x19c1, 0x19c7, 1}, + {0x1a00, 0x1a16, 1}, + {0x1a20, 0x1a54, 1}, + {0x1aa7, 0x1b05, 94}, + {0x1b06, 0x1b33, 1}, + {0x1b45, 0x1b4b, 1}, + {0x1b83, 0x1ba0, 1}, + {0x1bae, 0x1baf, 1}, + {0x1bc0, 0x1be5, 1}, + {0x1c00, 0x1c23, 1}, + {0x1c4d, 0x1c4f, 1}, + {0x1c5a, 0x1c7d, 1}, + {0x1ce9, 0x1cec, 1}, + {0x1cee, 0x1cf1, 1}, + {0x1d00, 0x1dbf, 1}, + {0x1e00, 0x1f15, 1}, + {0x1f18, 0x1f1d, 1}, + {0x1f20, 0x1f45, 1}, + {0x1f48, 0x1f4d, 1}, + {0x1f50, 0x1f57, 1}, + {0x1f59, 0x1f5f, 2}, + {0x1f60, 0x1f7d, 1}, + {0x1f80, 0x1fb4, 1}, + {0x1fb6, 0x1fbc, 1}, + {0x1fbe, 0x1fc2, 4}, + {0x1fc3, 0x1fc4, 1}, + {0x1fc6, 0x1fcc, 1}, + {0x1fd0, 0x1fd3, 1}, + {0x1fd6, 0x1fdb, 1}, + {0x1fe0, 0x1fec, 1}, + {0x1ff2, 0x1ff4, 1}, + {0x1ff6, 0x1ffc, 1}, + {0x2071, 0x207f, 14}, + {0x2090, 0x209c, 1}, + {0x2102, 0x2107, 5}, + {0x210a, 0x2113, 1}, + {0x2115, 0x2119, 4}, + {0x211a, 0x211d, 1}, + {0x2124, 0x212a, 2}, + {0x212b, 0x212d, 1}, + {0x212f, 0x2139, 1}, + {0x213c, 0x213f, 1}, + {0x2145, 0x2149, 1}, + {0x214e, 0x2183, 53}, + {0x2184, 0x2c00, 2684}, + {0x2c01, 0x2c2e, 1}, + {0x2c30, 0x2c5e, 1}, + {0x2c60, 0x2ce4, 1}, + {0x2ceb, 0x2cee, 1}, + {0x2d00, 0x2d25, 1}, + {0x2d30, 0x2d65, 1}, + {0x2d6f, 0x2d80, 17}, + {0x2d81, 0x2d96, 1}, + {0x2da0, 0x2da6, 1}, + {0x2da8, 0x2dae, 1}, + {0x2db0, 0x2db6, 1}, + {0x2db8, 0x2dbe, 1}, + {0x2dc0, 0x2dc6, 1}, + {0x2dc8, 0x2dce, 1}, + {0x2dd0, 0x2dd6, 1}, + {0x2dd8, 0x2dde, 1}, + {0x2e2f, 0x3005, 470}, + {0x3006, 0x3031, 43}, + {0x3032, 0x3035, 1}, + {0x303b, 0x303c, 1}, + {0x3041, 0x3096, 1}, + {0x309d, 0x309f, 1}, + {0x30a1, 0x30fa, 1}, + {0x30fc, 0x30ff, 1}, + {0x3105, 0x312d, 1}, + {0x3131, 0x318e, 1}, + {0x31a0, 0x31ba, 1}, + {0x31f0, 0x31ff, 1}, + {0x3400, 0x4db5, 1}, + {0x4e00, 0x9fcb, 1}, + {0xa000, 0xa48c, 1}, + {0xa4d0, 0xa4fd, 1}, + {0xa500, 0xa60c, 1}, + {0xa610, 0xa61f, 1}, + {0xa62a, 0xa62b, 1}, + {0xa640, 0xa66e, 1}, + {0xa67f, 0xa697, 1}, + {0xa6a0, 0xa6e5, 1}, + {0xa717, 0xa71f, 1}, + {0xa722, 0xa788, 1}, + {0xa78b, 0xa78e, 1}, + {0xa790, 0xa791, 1}, + {0xa7a0, 0xa7a9, 1}, + {0xa7fa, 0xa801, 1}, + {0xa803, 0xa805, 1}, + {0xa807, 0xa80a, 1}, + {0xa80c, 0xa822, 1}, + {0xa840, 0xa873, 1}, + {0xa882, 0xa8b3, 1}, + {0xa8f2, 0xa8f7, 1}, + {0xa8fb, 0xa90a, 15}, + {0xa90b, 0xa925, 1}, + {0xa930, 0xa946, 1}, + {0xa960, 0xa97c, 1}, + {0xa984, 0xa9b2, 1}, + {0xa9cf, 0xaa00, 49}, + {0xaa01, 0xaa28, 1}, + {0xaa40, 0xaa42, 1}, + {0xaa44, 0xaa4b, 1}, + {0xaa60, 0xaa76, 1}, + {0xaa7a, 0xaa80, 6}, + {0xaa81, 0xaaaf, 1}, + {0xaab1, 0xaab5, 4}, + {0xaab6, 0xaab9, 3}, + {0xaaba, 0xaabd, 1}, + {0xaac0, 0xaac2, 2}, + {0xaadb, 0xaadd, 1}, + {0xab01, 0xab06, 1}, + {0xab09, 0xab0e, 1}, + {0xab11, 0xab16, 1}, + {0xab20, 0xab26, 1}, + {0xab28, 0xab2e, 1}, + {0xabc0, 0xabe2, 1}, + {0xac00, 0xd7a3, 1}, + {0xd7b0, 0xd7c6, 1}, + {0xd7cb, 0xd7fb, 1}, + {0xf900, 0xfa2d, 1}, + {0xfa30, 0xfa6d, 1}, + {0xfa70, 0xfad9, 1}, + {0xfb00, 0xfb06, 1}, + {0xfb13, 0xfb17, 1}, + {0xfb1d, 0xfb1f, 2}, + {0xfb20, 0xfb28, 1}, + {0xfb2a, 0xfb36, 1}, + {0xfb38, 0xfb3c, 1}, + {0xfb3e, 0xfb40, 2}, + {0xfb41, 0xfb43, 2}, + {0xfb44, 0xfb46, 2}, + {0xfb47, 0xfbb1, 1}, + {0xfbd3, 0xfd3d, 1}, + {0xfd50, 0xfd8f, 1}, + {0xfd92, 0xfdc7, 1}, + {0xfdf0, 0xfdfb, 1}, + {0xfe70, 0xfe74, 1}, + {0xfe76, 0xfefc, 1}, + {0xff21, 0xff3a, 1}, + {0xff41, 0xff5a, 1}, + {0xff66, 0xffbe, 1}, + {0xffc2, 0xffc7, 1}, + {0xffca, 0xffcf, 1}, + {0xffd2, 0xffd7, 1}, + {0xffda, 0xffdc, 1}, + }, + R32: []Range32{ + {0x10000, 0x1000b, 1}, + {0x1000d, 0x10026, 1}, + {0x10028, 0x1003a, 1}, + {0x1003c, 0x1003d, 1}, + {0x1003f, 0x1004d, 1}, + {0x10050, 0x1005d, 1}, + {0x10080, 0x100fa, 1}, + {0x10280, 0x1029c, 1}, + {0x102a0, 0x102d0, 1}, + {0x10300, 0x1031e, 1}, + {0x10330, 0x10340, 1}, + {0x10342, 0x10349, 1}, + {0x10380, 0x1039d, 1}, + {0x103a0, 0x103c3, 1}, + {0x103c8, 0x103cf, 1}, + {0x10400, 0x1049d, 1}, + {0x10800, 0x10805, 1}, + {0x10808, 0x1080a, 2}, + {0x1080b, 0x10835, 1}, + {0x10837, 0x10838, 1}, + {0x1083c, 0x1083f, 3}, + {0x10840, 0x10855, 1}, + {0x10900, 0x10915, 1}, + {0x10920, 0x10939, 1}, + {0x10a00, 0x10a10, 16}, + {0x10a11, 0x10a13, 1}, + {0x10a15, 0x10a17, 1}, + {0x10a19, 0x10a33, 1}, + {0x10a60, 0x10a7c, 1}, + {0x10b00, 0x10b35, 1}, + {0x10b40, 0x10b55, 1}, + {0x10b60, 0x10b72, 1}, + {0x10c00, 0x10c48, 1}, + {0x11003, 0x11037, 1}, + {0x11083, 0x110af, 1}, + {0x12000, 0x1236e, 1}, + {0x13000, 0x1342e, 1}, + {0x16800, 0x16a38, 1}, + {0x1b000, 0x1b001, 1}, + {0x1d400, 0x1d454, 1}, + {0x1d456, 0x1d49c, 1}, + {0x1d49e, 0x1d49f, 1}, + {0x1d4a2, 0x1d4a5, 3}, + {0x1d4a6, 0x1d4a9, 3}, + {0x1d4aa, 0x1d4ac, 1}, + {0x1d4ae, 0x1d4b9, 1}, + {0x1d4bb, 0x1d4bd, 2}, + {0x1d4be, 0x1d4c3, 1}, + {0x1d4c5, 0x1d505, 1}, + {0x1d507, 0x1d50a, 1}, + {0x1d50d, 0x1d514, 1}, + {0x1d516, 0x1d51c, 1}, + {0x1d51e, 0x1d539, 1}, + {0x1d53b, 0x1d53e, 1}, + {0x1d540, 0x1d544, 1}, + {0x1d546, 0x1d54a, 4}, + {0x1d54b, 0x1d550, 1}, + {0x1d552, 0x1d6a5, 1}, + {0x1d6a8, 0x1d6c0, 1}, + {0x1d6c2, 0x1d6da, 1}, + {0x1d6dc, 0x1d6fa, 1}, + {0x1d6fc, 0x1d714, 1}, + {0x1d716, 0x1d734, 1}, + {0x1d736, 0x1d74e, 1}, + {0x1d750, 0x1d76e, 1}, + {0x1d770, 0x1d788, 1}, + {0x1d78a, 0x1d7a8, 1}, + {0x1d7aa, 0x1d7c2, 1}, + {0x1d7c4, 0x1d7cb, 1}, + {0x20000, 0x2a6d6, 1}, + {0x2a700, 0x2b734, 1}, + {0x2b740, 0x2b81d, 1}, + {0x2f800, 0x2fa1d, 1}, + }, +} + +var _N = &RangeTable{ + R16: []Range16{ + {0x0030, 0x0039, 1}, + {0x00b2, 0x00b3, 1}, + {0x00b9, 0x00bc, 3}, + {0x00bd, 0x00be, 1}, + {0x0660, 0x0669, 1}, + {0x06f0, 0x06f9, 1}, + {0x07c0, 0x07c9, 1}, + {0x0966, 0x096f, 1}, + {0x09e6, 0x09ef, 1}, + {0x09f4, 0x09f9, 1}, + {0x0a66, 0x0a6f, 1}, + {0x0ae6, 0x0aef, 1}, + {0x0b66, 0x0b6f, 1}, + {0x0b72, 0x0b77, 1}, + {0x0be6, 0x0bf2, 1}, + {0x0c66, 0x0c6f, 1}, + {0x0c78, 0x0c7e, 1}, + {0x0ce6, 0x0cef, 1}, + {0x0d66, 0x0d75, 1}, + {0x0e50, 0x0e59, 1}, + {0x0ed0, 0x0ed9, 1}, + {0x0f20, 0x0f33, 1}, + {0x1040, 0x1049, 1}, + {0x1090, 0x1099, 1}, + {0x1369, 0x137c, 1}, + {0x16ee, 0x16f0, 1}, + {0x17e0, 0x17e9, 1}, + {0x17f0, 0x17f9, 1}, + {0x1810, 0x1819, 1}, + {0x1946, 0x194f, 1}, + {0x19d0, 0x19da, 1}, + {0x1a80, 0x1a89, 1}, + {0x1a90, 0x1a99, 1}, + {0x1b50, 0x1b59, 1}, + {0x1bb0, 0x1bb9, 1}, + {0x1c40, 0x1c49, 1}, + {0x1c50, 0x1c59, 1}, + {0x2070, 0x2074, 4}, + {0x2075, 0x2079, 1}, + {0x2080, 0x2089, 1}, + {0x2150, 0x2182, 1}, + {0x2185, 0x2189, 1}, + {0x2460, 0x249b, 1}, + {0x24ea, 0x24ff, 1}, + {0x2776, 0x2793, 1}, + {0x2cfd, 0x3007, 778}, + {0x3021, 0x3029, 1}, + {0x3038, 0x303a, 1}, + {0x3192, 0x3195, 1}, + {0x3220, 0x3229, 1}, + {0x3251, 0x325f, 1}, + {0x3280, 0x3289, 1}, + {0x32b1, 0x32bf, 1}, + {0xa620, 0xa629, 1}, + {0xa6e6, 0xa6ef, 1}, + {0xa830, 0xa835, 1}, + {0xa8d0, 0xa8d9, 1}, + {0xa900, 0xa909, 1}, + {0xa9d0, 0xa9d9, 1}, + {0xaa50, 0xaa59, 1}, + {0xabf0, 0xabf9, 1}, + {0xff10, 0xff19, 1}, + }, + R32: []Range32{ + {0x10107, 0x10133, 1}, + {0x10140, 0x10178, 1}, + {0x1018a, 0x10320, 406}, + {0x10321, 0x10323, 1}, + {0x10341, 0x1034a, 9}, + {0x103d1, 0x103d5, 1}, + {0x104a0, 0x104a9, 1}, + {0x10858, 0x1085f, 1}, + {0x10916, 0x1091b, 1}, + {0x10a40, 0x10a47, 1}, + {0x10a7d, 0x10a7e, 1}, + {0x10b58, 0x10b5f, 1}, + {0x10b78, 0x10b7f, 1}, + {0x10e60, 0x10e7e, 1}, + {0x11052, 0x1106f, 1}, + {0x12400, 0x12462, 1}, + {0x1d360, 0x1d371, 1}, + {0x1d7ce, 0x1d7ff, 1}, + {0x1f100, 0x1f10a, 1}, + }, +} + +var _P = &RangeTable{ + R16: []Range16{ + {0x0021, 0x0023, 1}, + {0x0025, 0x002a, 1}, + {0x002c, 0x002f, 1}, + {0x003a, 0x003b, 1}, + {0x003f, 0x0040, 1}, + {0x005b, 0x005d, 1}, + {0x005f, 0x007b, 28}, + {0x007d, 0x00a1, 36}, + {0x00ab, 0x00b7, 12}, + {0x00bb, 0x00bf, 4}, + {0x037e, 0x0387, 9}, + {0x055a, 0x055f, 1}, + {0x0589, 0x058a, 1}, + {0x05be, 0x05c0, 2}, + {0x05c3, 0x05c6, 3}, + {0x05f3, 0x05f4, 1}, + {0x0609, 0x060a, 1}, + {0x060c, 0x060d, 1}, + {0x061b, 0x061e, 3}, + {0x061f, 0x066a, 75}, + {0x066b, 0x066d, 1}, + {0x06d4, 0x0700, 44}, + {0x0701, 0x070d, 1}, + {0x07f7, 0x07f9, 1}, + {0x0830, 0x083e, 1}, + {0x085e, 0x0964, 262}, + {0x0965, 0x0970, 11}, + {0x0df4, 0x0e4f, 91}, + {0x0e5a, 0x0e5b, 1}, + {0x0f04, 0x0f12, 1}, + {0x0f3a, 0x0f3d, 1}, + {0x0f85, 0x0fd0, 75}, + {0x0fd1, 0x0fd4, 1}, + {0x0fd9, 0x0fda, 1}, + {0x104a, 0x104f, 1}, + {0x10fb, 0x1361, 614}, + {0x1362, 0x1368, 1}, + {0x1400, 0x166d, 621}, + {0x166e, 0x169b, 45}, + {0x169c, 0x16eb, 79}, + {0x16ec, 0x16ed, 1}, + {0x1735, 0x1736, 1}, + {0x17d4, 0x17d6, 1}, + {0x17d8, 0x17da, 1}, + {0x1800, 0x180a, 1}, + {0x1944, 0x1945, 1}, + {0x1a1e, 0x1a1f, 1}, + {0x1aa0, 0x1aa6, 1}, + {0x1aa8, 0x1aad, 1}, + {0x1b5a, 0x1b60, 1}, + {0x1bfc, 0x1bff, 1}, + {0x1c3b, 0x1c3f, 1}, + {0x1c7e, 0x1c7f, 1}, + {0x1cd3, 0x2010, 829}, + {0x2011, 0x2027, 1}, + {0x2030, 0x2043, 1}, + {0x2045, 0x2051, 1}, + {0x2053, 0x205e, 1}, + {0x207d, 0x207e, 1}, + {0x208d, 0x208e, 1}, + {0x2329, 0x232a, 1}, + {0x2768, 0x2775, 1}, + {0x27c5, 0x27c6, 1}, + {0x27e6, 0x27ef, 1}, + {0x2983, 0x2998, 1}, + {0x29d8, 0x29db, 1}, + {0x29fc, 0x29fd, 1}, + {0x2cf9, 0x2cfc, 1}, + {0x2cfe, 0x2cff, 1}, + {0x2d70, 0x2e00, 144}, + {0x2e01, 0x2e2e, 1}, + {0x2e30, 0x2e31, 1}, + {0x3001, 0x3003, 1}, + {0x3008, 0x3011, 1}, + {0x3014, 0x301f, 1}, + {0x3030, 0x303d, 13}, + {0x30a0, 0x30fb, 91}, + {0xa4fe, 0xa4ff, 1}, + {0xa60d, 0xa60f, 1}, + {0xa673, 0xa67e, 11}, + {0xa6f2, 0xa6f7, 1}, + {0xa874, 0xa877, 1}, + {0xa8ce, 0xa8cf, 1}, + {0xa8f8, 0xa8fa, 1}, + {0xa92e, 0xa92f, 1}, + {0xa95f, 0xa9c1, 98}, + {0xa9c2, 0xa9cd, 1}, + {0xa9de, 0xa9df, 1}, + {0xaa5c, 0xaa5f, 1}, + {0xaade, 0xaadf, 1}, + {0xabeb, 0xfd3e, 20819}, + {0xfd3f, 0xfe10, 209}, + {0xfe11, 0xfe19, 1}, + {0xfe30, 0xfe52, 1}, + {0xfe54, 0xfe61, 1}, + {0xfe63, 0xfe68, 5}, + {0xfe6a, 0xfe6b, 1}, + {0xff01, 0xff03, 1}, + {0xff05, 0xff0a, 1}, + {0xff0c, 0xff0f, 1}, + {0xff1a, 0xff1b, 1}, + {0xff1f, 0xff20, 1}, + {0xff3b, 0xff3d, 1}, + {0xff3f, 0xff5b, 28}, + {0xff5d, 0xff5f, 2}, + {0xff60, 0xff65, 1}, + }, + R32: []Range32{ + {0x10100, 0x10101, 1}, + {0x1039f, 0x103d0, 49}, + {0x10857, 0x1091f, 200}, + {0x1093f, 0x10a50, 273}, + {0x10a51, 0x10a58, 1}, + {0x10a7f, 0x10b39, 186}, + {0x10b3a, 0x10b3f, 1}, + {0x11047, 0x1104d, 1}, + {0x110bb, 0x110bc, 1}, + {0x110be, 0x110c1, 1}, + {0x12470, 0x12473, 1}, + }, +} + +var _S = &RangeTable{ + R16: []Range16{ + {0x0024, 0x002b, 7}, + {0x003c, 0x003e, 1}, + {0x005e, 0x0060, 2}, + {0x007c, 0x007e, 2}, + {0x00a2, 0x00a9, 1}, + {0x00ac, 0x00ae, 2}, + {0x00af, 0x00b1, 1}, + {0x00b4, 0x00b8, 2}, + {0x00d7, 0x00f7, 32}, + {0x02c2, 0x02c5, 1}, + {0x02d2, 0x02df, 1}, + {0x02e5, 0x02eb, 1}, + {0x02ed, 0x02ef, 2}, + {0x02f0, 0x02ff, 1}, + {0x0375, 0x0384, 15}, + {0x0385, 0x03f6, 113}, + {0x0482, 0x0606, 388}, + {0x0607, 0x0608, 1}, + {0x060b, 0x060e, 3}, + {0x060f, 0x06de, 207}, + {0x06e9, 0x06fd, 20}, + {0x06fe, 0x07f6, 248}, + {0x09f2, 0x09f3, 1}, + {0x09fa, 0x09fb, 1}, + {0x0af1, 0x0b70, 127}, + {0x0bf3, 0x0bfa, 1}, + {0x0c7f, 0x0d79, 250}, + {0x0e3f, 0x0f01, 194}, + {0x0f02, 0x0f03, 1}, + {0x0f13, 0x0f17, 1}, + {0x0f1a, 0x0f1f, 1}, + {0x0f34, 0x0f38, 2}, + {0x0fbe, 0x0fc5, 1}, + {0x0fc7, 0x0fcc, 1}, + {0x0fce, 0x0fcf, 1}, + {0x0fd5, 0x0fd8, 1}, + {0x109e, 0x109f, 1}, + {0x1360, 0x1390, 48}, + {0x1391, 0x1399, 1}, + {0x17db, 0x1940, 357}, + {0x19de, 0x19ff, 1}, + {0x1b61, 0x1b6a, 1}, + {0x1b74, 0x1b7c, 1}, + {0x1fbd, 0x1fbf, 2}, + {0x1fc0, 0x1fc1, 1}, + {0x1fcd, 0x1fcf, 1}, + {0x1fdd, 0x1fdf, 1}, + {0x1fed, 0x1fef, 1}, + {0x1ffd, 0x1ffe, 1}, + {0x2044, 0x2052, 14}, + {0x207a, 0x207c, 1}, + {0x208a, 0x208c, 1}, + {0x20a0, 0x20b9, 1}, + {0x2100, 0x2101, 1}, + {0x2103, 0x2106, 1}, + {0x2108, 0x2109, 1}, + {0x2114, 0x2116, 2}, + {0x2117, 0x2118, 1}, + {0x211e, 0x2123, 1}, + {0x2125, 0x2129, 2}, + {0x212e, 0x213a, 12}, + {0x213b, 0x2140, 5}, + {0x2141, 0x2144, 1}, + {0x214a, 0x214d, 1}, + {0x214f, 0x2190, 65}, + {0x2191, 0x2328, 1}, + {0x232b, 0x23f3, 1}, + {0x2400, 0x2426, 1}, + {0x2440, 0x244a, 1}, + {0x249c, 0x24e9, 1}, + {0x2500, 0x26ff, 1}, + {0x2701, 0x2767, 1}, + {0x2794, 0x27c4, 1}, + {0x27c7, 0x27ca, 1}, + {0x27cc, 0x27ce, 2}, + {0x27cf, 0x27e5, 1}, + {0x27f0, 0x2982, 1}, + {0x2999, 0x29d7, 1}, + {0x29dc, 0x29fb, 1}, + {0x29fe, 0x2b4c, 1}, + {0x2b50, 0x2b59, 1}, + {0x2ce5, 0x2cea, 1}, + {0x2e80, 0x2e99, 1}, + {0x2e9b, 0x2ef3, 1}, + {0x2f00, 0x2fd5, 1}, + {0x2ff0, 0x2ffb, 1}, + {0x3004, 0x3012, 14}, + {0x3013, 0x3020, 13}, + {0x3036, 0x3037, 1}, + {0x303e, 0x303f, 1}, + {0x309b, 0x309c, 1}, + {0x3190, 0x3191, 1}, + {0x3196, 0x319f, 1}, + {0x31c0, 0x31e3, 1}, + {0x3200, 0x321e, 1}, + {0x322a, 0x3250, 1}, + {0x3260, 0x327f, 1}, + {0x328a, 0x32b0, 1}, + {0x32c0, 0x32fe, 1}, + {0x3300, 0x33ff, 1}, + {0x4dc0, 0x4dff, 1}, + {0xa490, 0xa4c6, 1}, + {0xa700, 0xa716, 1}, + {0xa720, 0xa721, 1}, + {0xa789, 0xa78a, 1}, + {0xa828, 0xa82b, 1}, + {0xa836, 0xa839, 1}, + {0xaa77, 0xaa79, 1}, + {0xfb29, 0xfbb2, 137}, + {0xfbb3, 0xfbc1, 1}, + {0xfdfc, 0xfdfd, 1}, + {0xfe62, 0xfe64, 2}, + {0xfe65, 0xfe66, 1}, + {0xfe69, 0xff04, 155}, + {0xff0b, 0xff1c, 17}, + {0xff1d, 0xff1e, 1}, + {0xff3e, 0xff40, 2}, + {0xff5c, 0xff5e, 2}, + {0xffe0, 0xffe6, 1}, + {0xffe8, 0xffee, 1}, + {0xfffc, 0xfffd, 1}, + }, + R32: []Range32{ + {0x10102, 0x10137, 53}, + {0x10138, 0x1013f, 1}, + {0x10179, 0x10189, 1}, + {0x10190, 0x1019b, 1}, + {0x101d0, 0x101fc, 1}, + {0x1d000, 0x1d0f5, 1}, + {0x1d100, 0x1d126, 1}, + {0x1d129, 0x1d164, 1}, + {0x1d16a, 0x1d16c, 1}, + {0x1d183, 0x1d184, 1}, + {0x1d18c, 0x1d1a9, 1}, + {0x1d1ae, 0x1d1dd, 1}, + {0x1d200, 0x1d241, 1}, + {0x1d245, 0x1d300, 187}, + {0x1d301, 0x1d356, 1}, + {0x1d6c1, 0x1d6db, 26}, + {0x1d6fb, 0x1d715, 26}, + {0x1d735, 0x1d74f, 26}, + {0x1d76f, 0x1d789, 26}, + {0x1d7a9, 0x1d7c3, 26}, + {0x1f000, 0x1f02b, 1}, + {0x1f030, 0x1f093, 1}, + {0x1f0a0, 0x1f0ae, 1}, + {0x1f0b1, 0x1f0be, 1}, + {0x1f0c1, 0x1f0cf, 1}, + {0x1f0d1, 0x1f0df, 1}, + {0x1f110, 0x1f12e, 1}, + {0x1f130, 0x1f169, 1}, + {0x1f170, 0x1f19a, 1}, + {0x1f1e6, 0x1f202, 1}, + {0x1f210, 0x1f23a, 1}, + {0x1f240, 0x1f248, 1}, + {0x1f250, 0x1f251, 1}, + {0x1f300, 0x1f320, 1}, + {0x1f330, 0x1f335, 1}, + {0x1f337, 0x1f37c, 1}, + {0x1f380, 0x1f393, 1}, + {0x1f3a0, 0x1f3c4, 1}, + {0x1f3c6, 0x1f3ca, 1}, + {0x1f3e0, 0x1f3f0, 1}, + {0x1f400, 0x1f43e, 1}, + {0x1f440, 0x1f442, 2}, + {0x1f443, 0x1f4f7, 1}, + {0x1f4f9, 0x1f4fc, 1}, + {0x1f500, 0x1f53d, 1}, + {0x1f550, 0x1f567, 1}, + {0x1f5fb, 0x1f5ff, 1}, + {0x1f601, 0x1f610, 1}, + {0x1f612, 0x1f614, 1}, + {0x1f616, 0x1f61c, 2}, + {0x1f61d, 0x1f61e, 1}, + {0x1f620, 0x1f625, 1}, + {0x1f628, 0x1f62b, 1}, + {0x1f62d, 0x1f630, 3}, + {0x1f631, 0x1f633, 1}, + {0x1f635, 0x1f640, 1}, + {0x1f645, 0x1f64f, 1}, + {0x1f680, 0x1f6c5, 1}, + {0x1f700, 0x1f773, 1}, + }, +} + +var _Z = &RangeTable{ + R16: []Range16{ + {0x0020, 0x00a0, 128}, + {0x1680, 0x180e, 398}, + {0x2000, 0x200a, 1}, + {0x2028, 0x2029, 1}, + {0x202f, 0x205f, 48}, + {0x3000, 0x3000, 1}, + }, +} + +var _Me = &RangeTable{ + R16: []Range16{ + {0x0488, 0x0489, 1}, + {0x20dd, 0x20e0, 1}, + {0x20e2, 0x20e4, 1}, + {0xa670, 0xa672, 1}, + }, +} + +var _Mc = &RangeTable{ + R16: []Range16{ + {0x0903, 0x093b, 56}, + {0x093e, 0x0940, 1}, + {0x0949, 0x094c, 1}, + {0x094e, 0x094f, 1}, + {0x0982, 0x0983, 1}, + {0x09be, 0x09c0, 1}, + {0x09c7, 0x09c8, 1}, + {0x09cb, 0x09cc, 1}, + {0x09d7, 0x0a03, 44}, + {0x0a3e, 0x0a40, 1}, + {0x0a83, 0x0abe, 59}, + {0x0abf, 0x0ac0, 1}, + {0x0ac9, 0x0acb, 2}, + {0x0acc, 0x0b02, 54}, + {0x0b03, 0x0b3e, 59}, + {0x0b40, 0x0b47, 7}, + {0x0b48, 0x0b4b, 3}, + {0x0b4c, 0x0b57, 11}, + {0x0bbe, 0x0bbf, 1}, + {0x0bc1, 0x0bc2, 1}, + {0x0bc6, 0x0bc8, 1}, + {0x0bca, 0x0bcc, 1}, + {0x0bd7, 0x0c01, 42}, + {0x0c02, 0x0c03, 1}, + {0x0c41, 0x0c44, 1}, + {0x0c82, 0x0c83, 1}, + {0x0cbe, 0x0cc0, 2}, + {0x0cc1, 0x0cc4, 1}, + {0x0cc7, 0x0cc8, 1}, + {0x0cca, 0x0ccb, 1}, + {0x0cd5, 0x0cd6, 1}, + {0x0d02, 0x0d03, 1}, + {0x0d3e, 0x0d40, 1}, + {0x0d46, 0x0d48, 1}, + {0x0d4a, 0x0d4c, 1}, + {0x0d57, 0x0d82, 43}, + {0x0d83, 0x0dcf, 76}, + {0x0dd0, 0x0dd1, 1}, + {0x0dd8, 0x0ddf, 1}, + {0x0df2, 0x0df3, 1}, + {0x0f3e, 0x0f3f, 1}, + {0x0f7f, 0x102b, 172}, + {0x102c, 0x1031, 5}, + {0x1038, 0x103b, 3}, + {0x103c, 0x1056, 26}, + {0x1057, 0x1062, 11}, + {0x1063, 0x1064, 1}, + {0x1067, 0x106d, 1}, + {0x1083, 0x1084, 1}, + {0x1087, 0x108c, 1}, + {0x108f, 0x109a, 11}, + {0x109b, 0x109c, 1}, + {0x17b6, 0x17be, 8}, + {0x17bf, 0x17c5, 1}, + {0x17c7, 0x17c8, 1}, + {0x1923, 0x1926, 1}, + {0x1929, 0x192b, 1}, + {0x1930, 0x1931, 1}, + {0x1933, 0x1938, 1}, + {0x19b0, 0x19c0, 1}, + {0x19c8, 0x19c9, 1}, + {0x1a19, 0x1a1b, 1}, + {0x1a55, 0x1a57, 2}, + {0x1a61, 0x1a63, 2}, + {0x1a64, 0x1a6d, 9}, + {0x1a6e, 0x1a72, 1}, + {0x1b04, 0x1b35, 49}, + {0x1b3b, 0x1b3d, 2}, + {0x1b3e, 0x1b41, 1}, + {0x1b43, 0x1b44, 1}, + {0x1b82, 0x1ba1, 31}, + {0x1ba6, 0x1ba7, 1}, + {0x1baa, 0x1be7, 61}, + {0x1bea, 0x1bec, 1}, + {0x1bee, 0x1bf2, 4}, + {0x1bf3, 0x1c24, 49}, + {0x1c25, 0x1c2b, 1}, + {0x1c34, 0x1c35, 1}, + {0x1ce1, 0x1cf2, 17}, + {0xa823, 0xa824, 1}, + {0xa827, 0xa880, 89}, + {0xa881, 0xa8b4, 51}, + {0xa8b5, 0xa8c3, 1}, + {0xa952, 0xa953, 1}, + {0xa983, 0xa9b4, 49}, + {0xa9b5, 0xa9ba, 5}, + {0xa9bb, 0xa9bd, 2}, + {0xa9be, 0xa9c0, 1}, + {0xaa2f, 0xaa30, 1}, + {0xaa33, 0xaa34, 1}, + {0xaa4d, 0xaa7b, 46}, + {0xabe3, 0xabe4, 1}, + {0xabe6, 0xabe7, 1}, + {0xabe9, 0xabea, 1}, + {0xabec, 0xabec, 1}, + }, + R32: []Range32{ + {0x11000, 0x11000, 1}, + {0x11002, 0x11082, 128}, + {0x110b0, 0x110b2, 1}, + {0x110b7, 0x110b8, 1}, + {0x1d165, 0x1d166, 1}, + {0x1d16d, 0x1d172, 1}, + }, +} + +var _Mn = &RangeTable{ + R16: []Range16{ + {0x0300, 0x036f, 1}, + {0x0483, 0x0487, 1}, + {0x0591, 0x05bd, 1}, + {0x05bf, 0x05c1, 2}, + {0x05c2, 0x05c4, 2}, + {0x05c5, 0x05c7, 2}, + {0x0610, 0x061a, 1}, + {0x064b, 0x065f, 1}, + {0x0670, 0x06d6, 102}, + {0x06d7, 0x06dc, 1}, + {0x06df, 0x06e4, 1}, + {0x06e7, 0x06e8, 1}, + {0x06ea, 0x06ed, 1}, + {0x0711, 0x0730, 31}, + {0x0731, 0x074a, 1}, + {0x07a6, 0x07b0, 1}, + {0x07eb, 0x07f3, 1}, + {0x0816, 0x0819, 1}, + {0x081b, 0x0823, 1}, + {0x0825, 0x0827, 1}, + {0x0829, 0x082d, 1}, + {0x0859, 0x085b, 1}, + {0x0900, 0x0902, 1}, + {0x093a, 0x093c, 2}, + {0x0941, 0x0948, 1}, + {0x094d, 0x0951, 4}, + {0x0952, 0x0957, 1}, + {0x0962, 0x0963, 1}, + {0x0981, 0x09bc, 59}, + {0x09c1, 0x09c4, 1}, + {0x09cd, 0x09e2, 21}, + {0x09e3, 0x0a01, 30}, + {0x0a02, 0x0a3c, 58}, + {0x0a41, 0x0a42, 1}, + {0x0a47, 0x0a48, 1}, + {0x0a4b, 0x0a4d, 1}, + {0x0a51, 0x0a70, 31}, + {0x0a71, 0x0a75, 4}, + {0x0a81, 0x0a82, 1}, + {0x0abc, 0x0ac1, 5}, + {0x0ac2, 0x0ac5, 1}, + {0x0ac7, 0x0ac8, 1}, + {0x0acd, 0x0ae2, 21}, + {0x0ae3, 0x0b01, 30}, + {0x0b3c, 0x0b3f, 3}, + {0x0b41, 0x0b44, 1}, + {0x0b4d, 0x0b56, 9}, + {0x0b62, 0x0b63, 1}, + {0x0b82, 0x0bc0, 62}, + {0x0bcd, 0x0c3e, 113}, + {0x0c3f, 0x0c40, 1}, + {0x0c46, 0x0c48, 1}, + {0x0c4a, 0x0c4d, 1}, + {0x0c55, 0x0c56, 1}, + {0x0c62, 0x0c63, 1}, + {0x0cbc, 0x0cbf, 3}, + {0x0cc6, 0x0ccc, 6}, + {0x0ccd, 0x0ce2, 21}, + {0x0ce3, 0x0d41, 94}, + {0x0d42, 0x0d44, 1}, + {0x0d4d, 0x0d62, 21}, + {0x0d63, 0x0dca, 103}, + {0x0dd2, 0x0dd4, 1}, + {0x0dd6, 0x0e31, 91}, + {0x0e34, 0x0e3a, 1}, + {0x0e47, 0x0e4e, 1}, + {0x0eb1, 0x0eb4, 3}, + {0x0eb5, 0x0eb9, 1}, + {0x0ebb, 0x0ebc, 1}, + {0x0ec8, 0x0ecd, 1}, + {0x0f18, 0x0f19, 1}, + {0x0f35, 0x0f39, 2}, + {0x0f71, 0x0f7e, 1}, + {0x0f80, 0x0f84, 1}, + {0x0f86, 0x0f87, 1}, + {0x0f8d, 0x0f97, 1}, + {0x0f99, 0x0fbc, 1}, + {0x0fc6, 0x102d, 103}, + {0x102e, 0x1030, 1}, + {0x1032, 0x1037, 1}, + {0x1039, 0x103a, 1}, + {0x103d, 0x103e, 1}, + {0x1058, 0x1059, 1}, + {0x105e, 0x1060, 1}, + {0x1071, 0x1074, 1}, + {0x1082, 0x1085, 3}, + {0x1086, 0x108d, 7}, + {0x109d, 0x135d, 704}, + {0x135e, 0x135f, 1}, + {0x1712, 0x1714, 1}, + {0x1732, 0x1734, 1}, + {0x1752, 0x1753, 1}, + {0x1772, 0x1773, 1}, + {0x17b7, 0x17bd, 1}, + {0x17c6, 0x17c9, 3}, + {0x17ca, 0x17d3, 1}, + {0x17dd, 0x180b, 46}, + {0x180c, 0x180d, 1}, + {0x18a9, 0x1920, 119}, + {0x1921, 0x1922, 1}, + {0x1927, 0x1928, 1}, + {0x1932, 0x1939, 7}, + {0x193a, 0x193b, 1}, + {0x1a17, 0x1a18, 1}, + {0x1a56, 0x1a58, 2}, + {0x1a59, 0x1a5e, 1}, + {0x1a60, 0x1a62, 2}, + {0x1a65, 0x1a6c, 1}, + {0x1a73, 0x1a7c, 1}, + {0x1a7f, 0x1b00, 129}, + {0x1b01, 0x1b03, 1}, + {0x1b34, 0x1b36, 2}, + {0x1b37, 0x1b3a, 1}, + {0x1b3c, 0x1b42, 6}, + {0x1b6b, 0x1b73, 1}, + {0x1b80, 0x1b81, 1}, + {0x1ba2, 0x1ba5, 1}, + {0x1ba8, 0x1ba9, 1}, + {0x1be6, 0x1be8, 2}, + {0x1be9, 0x1bed, 4}, + {0x1bef, 0x1bf1, 1}, + {0x1c2c, 0x1c33, 1}, + {0x1c36, 0x1c37, 1}, + {0x1cd0, 0x1cd2, 1}, + {0x1cd4, 0x1ce0, 1}, + {0x1ce2, 0x1ce8, 1}, + {0x1ced, 0x1dc0, 211}, + {0x1dc1, 0x1de6, 1}, + {0x1dfc, 0x1dff, 1}, + {0x20d0, 0x20dc, 1}, + {0x20e1, 0x20e5, 4}, + {0x20e6, 0x20f0, 1}, + {0x2cef, 0x2cf1, 1}, + {0x2d7f, 0x2de0, 97}, + {0x2de1, 0x2dff, 1}, + {0x302a, 0x302f, 1}, + {0x3099, 0x309a, 1}, + {0xa66f, 0xa67c, 13}, + {0xa67d, 0xa6f0, 115}, + {0xa6f1, 0xa802, 273}, + {0xa806, 0xa80b, 5}, + {0xa825, 0xa826, 1}, + {0xa8c4, 0xa8e0, 28}, + {0xa8e1, 0xa8f1, 1}, + {0xa926, 0xa92d, 1}, + {0xa947, 0xa951, 1}, + {0xa980, 0xa982, 1}, + {0xa9b3, 0xa9b6, 3}, + {0xa9b7, 0xa9b9, 1}, + {0xa9bc, 0xaa29, 109}, + {0xaa2a, 0xaa2e, 1}, + {0xaa31, 0xaa32, 1}, + {0xaa35, 0xaa36, 1}, + {0xaa43, 0xaa4c, 9}, + {0xaab0, 0xaab2, 2}, + {0xaab3, 0xaab4, 1}, + {0xaab7, 0xaab8, 1}, + {0xaabe, 0xaabf, 1}, + {0xaac1, 0xabe5, 292}, + {0xabe8, 0xabed, 5}, + {0xfb1e, 0xfe00, 738}, + {0xfe01, 0xfe0f, 1}, + {0xfe20, 0xfe26, 1}, + }, + R32: []Range32{ + {0x101fd, 0x10a01, 2052}, + {0x10a02, 0x10a03, 1}, + {0x10a05, 0x10a06, 1}, + {0x10a0c, 0x10a0f, 1}, + {0x10a38, 0x10a3a, 1}, + {0x10a3f, 0x11001, 1474}, + {0x11038, 0x11046, 1}, + {0x11080, 0x11081, 1}, + {0x110b3, 0x110b6, 1}, + {0x110b9, 0x110ba, 1}, + {0x1d167, 0x1d169, 1}, + {0x1d17b, 0x1d182, 1}, + {0x1d185, 0x1d18b, 1}, + {0x1d1aa, 0x1d1ad, 1}, + {0x1d242, 0x1d244, 1}, + {0xe0100, 0xe01ef, 1}, + }, +} + +var _Zl = &RangeTable{ + R16: []Range16{ + {0x2028, 0x2028, 1}, + }, +} + +var _Zp = &RangeTable{ + R16: []Range16{ + {0x2029, 0x2029, 1}, + }, +} + +var _Zs = &RangeTable{ + R16: []Range16{ + {0x0020, 0x00a0, 128}, + {0x1680, 0x180e, 398}, + {0x2000, 0x200a, 1}, + {0x202f, 0x205f, 48}, + {0x3000, 0x3000, 1}, + }, +} + +var _Cs = &RangeTable{ + R16: []Range16{ + {0xd800, 0xdfff, 1}, + }, +} + +var _Co = &RangeTable{ + R16: []Range16{ + {0xe000, 0xf8ff, 1}, + }, + R32: []Range32{ + {0xf0000, 0xffffd, 1}, + {0x100000, 0x10fffd, 1}, + }, +} + +var _Cf = &RangeTable{ + R16: []Range16{ + {0x00ad, 0x0600, 1363}, + {0x0601, 0x0603, 1}, + {0x06dd, 0x070f, 50}, + {0x17b4, 0x17b5, 1}, + {0x200b, 0x200f, 1}, + {0x202a, 0x202e, 1}, + {0x2060, 0x2064, 1}, + {0x206a, 0x206f, 1}, + {0xfeff, 0xfff9, 250}, + {0xfffa, 0xfffb, 1}, + }, + R32: []Range32{ + {0x110bd, 0x1d173, 49334}, + {0x1d174, 0x1d17a, 1}, + {0xe0001, 0xe0020, 31}, + {0xe0021, 0xe007f, 1}, + }, +} + +var _Cc = &RangeTable{ + R16: []Range16{ + {0x0001, 0x001f, 1}, + {0x007f, 0x009f, 1}, + }, +} + +var _Po = &RangeTable{ + R16: []Range16{ + {0x0021, 0x0023, 1}, + {0x0025, 0x0027, 1}, + {0x002a, 0x002e, 2}, + {0x002f, 0x003a, 11}, + {0x003b, 0x003f, 4}, + {0x0040, 0x005c, 28}, + {0x00a1, 0x00b7, 22}, + {0x00bf, 0x037e, 703}, + {0x0387, 0x055a, 467}, + {0x055b, 0x055f, 1}, + {0x0589, 0x05c0, 55}, + {0x05c3, 0x05c6, 3}, + {0x05f3, 0x05f4, 1}, + {0x0609, 0x060a, 1}, + {0x060c, 0x060d, 1}, + {0x061b, 0x061e, 3}, + {0x061f, 0x066a, 75}, + {0x066b, 0x066d, 1}, + {0x06d4, 0x0700, 44}, + {0x0701, 0x070d, 1}, + {0x07f7, 0x07f9, 1}, + {0x0830, 0x083e, 1}, + {0x085e, 0x0964, 262}, + {0x0965, 0x0970, 11}, + {0x0df4, 0x0e4f, 91}, + {0x0e5a, 0x0e5b, 1}, + {0x0f04, 0x0f12, 1}, + {0x0f85, 0x0fd0, 75}, + {0x0fd1, 0x0fd4, 1}, + {0x0fd9, 0x0fda, 1}, + {0x104a, 0x104f, 1}, + {0x10fb, 0x1361, 614}, + {0x1362, 0x1368, 1}, + {0x166d, 0x166e, 1}, + {0x16eb, 0x16ed, 1}, + {0x1735, 0x1736, 1}, + {0x17d4, 0x17d6, 1}, + {0x17d8, 0x17da, 1}, + {0x1800, 0x1805, 1}, + {0x1807, 0x180a, 1}, + {0x1944, 0x1945, 1}, + {0x1a1e, 0x1a1f, 1}, + {0x1aa0, 0x1aa6, 1}, + {0x1aa8, 0x1aad, 1}, + {0x1b5a, 0x1b60, 1}, + {0x1bfc, 0x1bff, 1}, + {0x1c3b, 0x1c3f, 1}, + {0x1c7e, 0x1c7f, 1}, + {0x1cd3, 0x2016, 835}, + {0x2017, 0x2020, 9}, + {0x2021, 0x2027, 1}, + {0x2030, 0x2038, 1}, + {0x203b, 0x203e, 1}, + {0x2041, 0x2043, 1}, + {0x2047, 0x2051, 1}, + {0x2053, 0x2055, 2}, + {0x2056, 0x205e, 1}, + {0x2cf9, 0x2cfc, 1}, + {0x2cfe, 0x2cff, 1}, + {0x2d70, 0x2e00, 144}, + {0x2e01, 0x2e06, 5}, + {0x2e07, 0x2e08, 1}, + {0x2e0b, 0x2e0e, 3}, + {0x2e0f, 0x2e16, 1}, + {0x2e18, 0x2e19, 1}, + {0x2e1b, 0x2e1e, 3}, + {0x2e1f, 0x2e2a, 11}, + {0x2e2b, 0x2e2e, 1}, + {0x2e30, 0x2e31, 1}, + {0x3001, 0x3003, 1}, + {0x303d, 0x30fb, 190}, + {0xa4fe, 0xa4ff, 1}, + {0xa60d, 0xa60f, 1}, + {0xa673, 0xa67e, 11}, + {0xa6f2, 0xa6f7, 1}, + {0xa874, 0xa877, 1}, + {0xa8ce, 0xa8cf, 1}, + {0xa8f8, 0xa8fa, 1}, + {0xa92e, 0xa92f, 1}, + {0xa95f, 0xa9c1, 98}, + {0xa9c2, 0xa9cd, 1}, + {0xa9de, 0xa9df, 1}, + {0xaa5c, 0xaa5f, 1}, + {0xaade, 0xaadf, 1}, + {0xabeb, 0xfe10, 21029}, + {0xfe11, 0xfe16, 1}, + {0xfe19, 0xfe30, 23}, + {0xfe45, 0xfe46, 1}, + {0xfe49, 0xfe4c, 1}, + {0xfe50, 0xfe52, 1}, + {0xfe54, 0xfe57, 1}, + {0xfe5f, 0xfe61, 1}, + {0xfe68, 0xfe6a, 2}, + {0xfe6b, 0xff01, 150}, + {0xff02, 0xff03, 1}, + {0xff05, 0xff07, 1}, + {0xff0a, 0xff0e, 2}, + {0xff0f, 0xff1a, 11}, + {0xff1b, 0xff1f, 4}, + {0xff20, 0xff3c, 28}, + {0xff61, 0xff64, 3}, + {0xff65, 0xff65, 1}, + }, + R32: []Range32{ + {0x10100, 0x10100, 1}, + {0x10101, 0x1039f, 670}, + {0x103d0, 0x10857, 1159}, + {0x1091f, 0x1093f, 32}, + {0x10a50, 0x10a58, 1}, + {0x10a7f, 0x10b39, 186}, + {0x10b3a, 0x10b3f, 1}, + {0x11047, 0x1104d, 1}, + {0x110bb, 0x110bc, 1}, + {0x110be, 0x110c1, 1}, + {0x12470, 0x12473, 1}, + }, +} + +var _Pi = &RangeTable{ + R16: []Range16{ + {0x00ab, 0x2018, 8045}, + {0x201b, 0x201c, 1}, + {0x201f, 0x2039, 26}, + {0x2e02, 0x2e04, 2}, + {0x2e09, 0x2e0c, 3}, + {0x2e1c, 0x2e20, 4}, + }, +} + +var _Pf = &RangeTable{ + R16: []Range16{ + {0x00bb, 0x2019, 8030}, + {0x201d, 0x203a, 29}, + {0x2e03, 0x2e05, 2}, + {0x2e0a, 0x2e0d, 3}, + {0x2e1d, 0x2e21, 4}, + }, +} + +var _Pe = &RangeTable{ + R16: []Range16{ + {0x0029, 0x005d, 52}, + {0x007d, 0x0f3b, 3774}, + {0x0f3d, 0x169c, 1887}, + {0x2046, 0x207e, 56}, + {0x208e, 0x232a, 668}, + {0x2769, 0x2775, 2}, + {0x27c6, 0x27e7, 33}, + {0x27e9, 0x27ef, 2}, + {0x2984, 0x2998, 2}, + {0x29d9, 0x29db, 2}, + {0x29fd, 0x2e23, 1062}, + {0x2e25, 0x2e29, 2}, + {0x3009, 0x3011, 2}, + {0x3015, 0x301b, 2}, + {0x301e, 0x301f, 1}, + {0xfd3f, 0xfe18, 217}, + {0xfe36, 0xfe44, 2}, + {0xfe48, 0xfe5a, 18}, + {0xfe5c, 0xfe5e, 2}, + {0xff09, 0xff3d, 52}, + {0xff5d, 0xff63, 3}, + }, +} + +var _Pd = &RangeTable{ + R16: []Range16{ + {0x002d, 0x058a, 1373}, + {0x05be, 0x1400, 3650}, + {0x1806, 0x2010, 2058}, + {0x2011, 0x2015, 1}, + {0x2e17, 0x2e1a, 3}, + {0x301c, 0x3030, 20}, + {0x30a0, 0xfe31, 52625}, + {0xfe32, 0xfe58, 38}, + {0xfe63, 0xff0d, 170}, + }, +} + +var _Pc = &RangeTable{ + R16: []Range16{ + {0x005f, 0x203f, 8160}, + {0x2040, 0x2054, 20}, + {0xfe33, 0xfe34, 1}, + {0xfe4d, 0xfe4f, 1}, + {0xff3f, 0xff3f, 1}, + }, +} + +var _Ps = &RangeTable{ + R16: []Range16{ + {0x0028, 0x005b, 51}, + {0x007b, 0x0f3a, 3775}, + {0x0f3c, 0x169b, 1887}, + {0x201a, 0x201e, 4}, + {0x2045, 0x207d, 56}, + {0x208d, 0x2329, 668}, + {0x2768, 0x2774, 2}, + {0x27c5, 0x27e6, 33}, + {0x27e8, 0x27ee, 2}, + {0x2983, 0x2997, 2}, + {0x29d8, 0x29da, 2}, + {0x29fc, 0x2e22, 1062}, + {0x2e24, 0x2e28, 2}, + {0x3008, 0x3010, 2}, + {0x3014, 0x301a, 2}, + {0x301d, 0xfd3e, 52513}, + {0xfe17, 0xfe35, 30}, + {0xfe37, 0xfe43, 2}, + {0xfe47, 0xfe59, 18}, + {0xfe5b, 0xfe5d, 2}, + {0xff08, 0xff3b, 51}, + {0xff5b, 0xff5f, 4}, + {0xff62, 0xff62, 1}, + }, +} + +var _Nd = &RangeTable{ + R16: []Range16{ + {0x0030, 0x0039, 1}, + {0x0660, 0x0669, 1}, + {0x06f0, 0x06f9, 1}, + {0x07c0, 0x07c9, 1}, + {0x0966, 0x096f, 1}, + {0x09e6, 0x09ef, 1}, + {0x0a66, 0x0a6f, 1}, + {0x0ae6, 0x0aef, 1}, + {0x0b66, 0x0b6f, 1}, + {0x0be6, 0x0bef, 1}, + {0x0c66, 0x0c6f, 1}, + {0x0ce6, 0x0cef, 1}, + {0x0d66, 0x0d6f, 1}, + {0x0e50, 0x0e59, 1}, + {0x0ed0, 0x0ed9, 1}, + {0x0f20, 0x0f29, 1}, + {0x1040, 0x1049, 1}, + {0x1090, 0x1099, 1}, + {0x17e0, 0x17e9, 1}, + {0x1810, 0x1819, 1}, + {0x1946, 0x194f, 1}, + {0x19d0, 0x19d9, 1}, + {0x1a80, 0x1a89, 1}, + {0x1a90, 0x1a99, 1}, + {0x1b50, 0x1b59, 1}, + {0x1bb0, 0x1bb9, 1}, + {0x1c40, 0x1c49, 1}, + {0x1c50, 0x1c59, 1}, + {0xa620, 0xa629, 1}, + {0xa8d0, 0xa8d9, 1}, + {0xa900, 0xa909, 1}, + {0xa9d0, 0xa9d9, 1}, + {0xaa50, 0xaa59, 1}, + {0xabf0, 0xabf9, 1}, + {0xff10, 0xff19, 1}, + }, + R32: []Range32{ + {0x104a0, 0x104a9, 1}, + {0x11066, 0x1106f, 1}, + {0x1d7ce, 0x1d7ff, 1}, + }, +} + +var _Nl = &RangeTable{ + R16: []Range16{ + {0x16ee, 0x16f0, 1}, + {0x2160, 0x2182, 1}, + {0x2185, 0x2188, 1}, + {0x3007, 0x3021, 26}, + {0x3022, 0x3029, 1}, + {0x3038, 0x303a, 1}, + {0xa6e6, 0xa6ef, 1}, + }, + R32: []Range32{ + {0x10140, 0x10174, 1}, + {0x10341, 0x1034a, 9}, + {0x103d1, 0x103d5, 1}, + {0x12400, 0x12462, 1}, + }, +} + +var _No = &RangeTable{ + R16: []Range16{ + {0x00b2, 0x00b3, 1}, + {0x00b9, 0x00bc, 3}, + {0x00bd, 0x00be, 1}, + {0x09f4, 0x09f9, 1}, + {0x0b72, 0x0b77, 1}, + {0x0bf0, 0x0bf2, 1}, + {0x0c78, 0x0c7e, 1}, + {0x0d70, 0x0d75, 1}, + {0x0f2a, 0x0f33, 1}, + {0x1369, 0x137c, 1}, + {0x17f0, 0x17f9, 1}, + {0x19da, 0x2070, 1686}, + {0x2074, 0x2079, 1}, + {0x2080, 0x2089, 1}, + {0x2150, 0x215f, 1}, + {0x2189, 0x2460, 727}, + {0x2461, 0x249b, 1}, + {0x24ea, 0x24ff, 1}, + {0x2776, 0x2793, 1}, + {0x2cfd, 0x3192, 1173}, + {0x3193, 0x3195, 1}, + {0x3220, 0x3229, 1}, + {0x3251, 0x325f, 1}, + {0x3280, 0x3289, 1}, + {0x32b1, 0x32bf, 1}, + {0xa830, 0xa835, 1}, + }, + R32: []Range32{ + {0x10107, 0x10133, 1}, + {0x10175, 0x10178, 1}, + {0x1018a, 0x10320, 406}, + {0x10321, 0x10323, 1}, + {0x10858, 0x1085f, 1}, + {0x10916, 0x1091b, 1}, + {0x10a40, 0x10a47, 1}, + {0x10a7d, 0x10a7e, 1}, + {0x10b58, 0x10b5f, 1}, + {0x10b78, 0x10b7f, 1}, + {0x10e60, 0x10e7e, 1}, + {0x11052, 0x11065, 1}, + {0x1d360, 0x1d371, 1}, + {0x1f100, 0x1f10a, 1}, + }, +} + +var _So = &RangeTable{ + R16: []Range16{ + {0x00a6, 0x00a7, 1}, + {0x00a9, 0x00ae, 5}, + {0x00b0, 0x00b6, 6}, + {0x0482, 0x060e, 396}, + {0x060f, 0x06de, 207}, + {0x06e9, 0x06fd, 20}, + {0x06fe, 0x07f6, 248}, + {0x09fa, 0x0b70, 374}, + {0x0bf3, 0x0bf8, 1}, + {0x0bfa, 0x0c7f, 133}, + {0x0d79, 0x0f01, 392}, + {0x0f02, 0x0f03, 1}, + {0x0f13, 0x0f17, 1}, + {0x0f1a, 0x0f1f, 1}, + {0x0f34, 0x0f38, 2}, + {0x0fbe, 0x0fc5, 1}, + {0x0fc7, 0x0fcc, 1}, + {0x0fce, 0x0fcf, 1}, + {0x0fd5, 0x0fd8, 1}, + {0x109e, 0x109f, 1}, + {0x1360, 0x1390, 48}, + {0x1391, 0x1399, 1}, + {0x1940, 0x19de, 158}, + {0x19df, 0x19ff, 1}, + {0x1b61, 0x1b6a, 1}, + {0x1b74, 0x1b7c, 1}, + {0x2100, 0x2101, 1}, + {0x2103, 0x2106, 1}, + {0x2108, 0x2109, 1}, + {0x2114, 0x2116, 2}, + {0x2117, 0x211e, 7}, + {0x211f, 0x2123, 1}, + {0x2125, 0x2129, 2}, + {0x212e, 0x213a, 12}, + {0x213b, 0x214a, 15}, + {0x214c, 0x214d, 1}, + {0x214f, 0x2195, 70}, + {0x2196, 0x2199, 1}, + {0x219c, 0x219f, 1}, + {0x21a1, 0x21a2, 1}, + {0x21a4, 0x21a5, 1}, + {0x21a7, 0x21ad, 1}, + {0x21af, 0x21cd, 1}, + {0x21d0, 0x21d1, 1}, + {0x21d3, 0x21d5, 2}, + {0x21d6, 0x21f3, 1}, + {0x2300, 0x2307, 1}, + {0x230c, 0x231f, 1}, + {0x2322, 0x2328, 1}, + {0x232b, 0x237b, 1}, + {0x237d, 0x239a, 1}, + {0x23b4, 0x23db, 1}, + {0x23e2, 0x23f3, 1}, + {0x2400, 0x2426, 1}, + {0x2440, 0x244a, 1}, + {0x249c, 0x24e9, 1}, + {0x2500, 0x25b6, 1}, + {0x25b8, 0x25c0, 1}, + {0x25c2, 0x25f7, 1}, + {0x2600, 0x266e, 1}, + {0x2670, 0x26ff, 1}, + {0x2701, 0x2767, 1}, + {0x2794, 0x27bf, 1}, + {0x2800, 0x28ff, 1}, + {0x2b00, 0x2b2f, 1}, + {0x2b45, 0x2b46, 1}, + {0x2b50, 0x2b59, 1}, + {0x2ce5, 0x2cea, 1}, + {0x2e80, 0x2e99, 1}, + {0x2e9b, 0x2ef3, 1}, + {0x2f00, 0x2fd5, 1}, + {0x2ff0, 0x2ffb, 1}, + {0x3004, 0x3012, 14}, + {0x3013, 0x3020, 13}, + {0x3036, 0x3037, 1}, + {0x303e, 0x303f, 1}, + {0x3190, 0x3191, 1}, + {0x3196, 0x319f, 1}, + {0x31c0, 0x31e3, 1}, + {0x3200, 0x321e, 1}, + {0x322a, 0x3250, 1}, + {0x3260, 0x327f, 1}, + {0x328a, 0x32b0, 1}, + {0x32c0, 0x32fe, 1}, + {0x3300, 0x33ff, 1}, + {0x4dc0, 0x4dff, 1}, + {0xa490, 0xa4c6, 1}, + {0xa828, 0xa82b, 1}, + {0xa836, 0xa837, 1}, + {0xa839, 0xaa77, 574}, + {0xaa78, 0xaa79, 1}, + {0xfdfd, 0xffe4, 487}, + {0xffe8, 0xffed, 5}, + {0xffee, 0xfffc, 14}, + {0xfffd, 0xfffd, 1}, + }, + R32: []Range32{ + {0x10102, 0x10102, 1}, + {0x10137, 0x1013f, 1}, + {0x10179, 0x10189, 1}, + {0x10190, 0x1019b, 1}, + {0x101d0, 0x101fc, 1}, + {0x1d000, 0x1d0f5, 1}, + {0x1d100, 0x1d126, 1}, + {0x1d129, 0x1d164, 1}, + {0x1d16a, 0x1d16c, 1}, + {0x1d183, 0x1d184, 1}, + {0x1d18c, 0x1d1a9, 1}, + {0x1d1ae, 0x1d1dd, 1}, + {0x1d200, 0x1d241, 1}, + {0x1d245, 0x1d300, 187}, + {0x1d301, 0x1d356, 1}, + {0x1f000, 0x1f02b, 1}, + {0x1f030, 0x1f093, 1}, + {0x1f0a0, 0x1f0ae, 1}, + {0x1f0b1, 0x1f0be, 1}, + {0x1f0c1, 0x1f0cf, 1}, + {0x1f0d1, 0x1f0df, 1}, + {0x1f110, 0x1f12e, 1}, + {0x1f130, 0x1f169, 1}, + {0x1f170, 0x1f19a, 1}, + {0x1f1e6, 0x1f202, 1}, + {0x1f210, 0x1f23a, 1}, + {0x1f240, 0x1f248, 1}, + {0x1f250, 0x1f251, 1}, + {0x1f300, 0x1f320, 1}, + {0x1f330, 0x1f335, 1}, + {0x1f337, 0x1f37c, 1}, + {0x1f380, 0x1f393, 1}, + {0x1f3a0, 0x1f3c4, 1}, + {0x1f3c6, 0x1f3ca, 1}, + {0x1f3e0, 0x1f3f0, 1}, + {0x1f400, 0x1f43e, 1}, + {0x1f440, 0x1f442, 2}, + {0x1f443, 0x1f4f7, 1}, + {0x1f4f9, 0x1f4fc, 1}, + {0x1f500, 0x1f53d, 1}, + {0x1f550, 0x1f567, 1}, + {0x1f5fb, 0x1f5ff, 1}, + {0x1f601, 0x1f610, 1}, + {0x1f612, 0x1f614, 1}, + {0x1f616, 0x1f61c, 2}, + {0x1f61d, 0x1f61e, 1}, + {0x1f620, 0x1f625, 1}, + {0x1f628, 0x1f62b, 1}, + {0x1f62d, 0x1f630, 3}, + {0x1f631, 0x1f633, 1}, + {0x1f635, 0x1f640, 1}, + {0x1f645, 0x1f64f, 1}, + {0x1f680, 0x1f6c5, 1}, + {0x1f700, 0x1f773, 1}, + }, +} + +var _Sm = &RangeTable{ + R16: []Range16{ + {0x002b, 0x003c, 17}, + {0x003d, 0x003e, 1}, + {0x007c, 0x007e, 2}, + {0x00ac, 0x00b1, 5}, + {0x00d7, 0x00f7, 32}, + {0x03f6, 0x0606, 528}, + {0x0607, 0x0608, 1}, + {0x2044, 0x2052, 14}, + {0x207a, 0x207c, 1}, + {0x208a, 0x208c, 1}, + {0x2118, 0x2140, 40}, + {0x2141, 0x2144, 1}, + {0x214b, 0x2190, 69}, + {0x2191, 0x2194, 1}, + {0x219a, 0x219b, 1}, + {0x21a0, 0x21a6, 3}, + {0x21ae, 0x21ce, 32}, + {0x21cf, 0x21d2, 3}, + {0x21d4, 0x21f4, 32}, + {0x21f5, 0x22ff, 1}, + {0x2308, 0x230b, 1}, + {0x2320, 0x2321, 1}, + {0x237c, 0x239b, 31}, + {0x239c, 0x23b3, 1}, + {0x23dc, 0x23e1, 1}, + {0x25b7, 0x25c1, 10}, + {0x25f8, 0x25ff, 1}, + {0x266f, 0x27c0, 337}, + {0x27c1, 0x27c4, 1}, + {0x27c7, 0x27ca, 1}, + {0x27cc, 0x27ce, 2}, + {0x27cf, 0x27e5, 1}, + {0x27f0, 0x27ff, 1}, + {0x2900, 0x2982, 1}, + {0x2999, 0x29d7, 1}, + {0x29dc, 0x29fb, 1}, + {0x29fe, 0x2aff, 1}, + {0x2b30, 0x2b44, 1}, + {0x2b47, 0x2b4c, 1}, + {0xfb29, 0xfe62, 825}, + {0xfe64, 0xfe66, 1}, + {0xff0b, 0xff1c, 17}, + {0xff1d, 0xff1e, 1}, + {0xff5c, 0xff5e, 2}, + {0xffe2, 0xffe9, 7}, + {0xffea, 0xffec, 1}, + }, + R32: []Range32{ + {0x1d6c1, 0x1d6db, 26}, + {0x1d6fb, 0x1d715, 26}, + {0x1d735, 0x1d74f, 26}, + {0x1d76f, 0x1d789, 26}, + {0x1d7a9, 0x1d7c3, 26}, + }, +} + +var _Sk = &RangeTable{ + R16: []Range16{ + {0x005e, 0x0060, 2}, + {0x00a8, 0x00af, 7}, + {0x00b4, 0x00b8, 4}, + {0x02c2, 0x02c5, 1}, + {0x02d2, 0x02df, 1}, + {0x02e5, 0x02eb, 1}, + {0x02ed, 0x02ef, 2}, + {0x02f0, 0x02ff, 1}, + {0x0375, 0x0384, 15}, + {0x0385, 0x1fbd, 7224}, + {0x1fbf, 0x1fc1, 1}, + {0x1fcd, 0x1fcf, 1}, + {0x1fdd, 0x1fdf, 1}, + {0x1fed, 0x1fef, 1}, + {0x1ffd, 0x1ffe, 1}, + {0x309b, 0x309c, 1}, + {0xa700, 0xa716, 1}, + {0xa720, 0xa721, 1}, + {0xa789, 0xa78a, 1}, + {0xfbb2, 0xfbc1, 1}, + {0xff3e, 0xff40, 2}, + {0xffe3, 0xffe3, 1}, + }, +} + +var _Sc = &RangeTable{ + R16: []Range16{ + {0x0024, 0x00a2, 126}, + {0x00a3, 0x00a5, 1}, + {0x060b, 0x09f2, 999}, + {0x09f3, 0x09fb, 8}, + {0x0af1, 0x0bf9, 264}, + {0x0e3f, 0x17db, 2460}, + {0x20a0, 0x20b9, 1}, + {0xa838, 0xfdfc, 21956}, + {0xfe69, 0xff04, 155}, + {0xffe0, 0xffe1, 1}, + {0xffe5, 0xffe6, 1}, + }, +} + +var _Lu = &RangeTable{ + R16: []Range16{ + {0x0041, 0x005a, 1}, + {0x00c0, 0x00d6, 1}, + {0x00d8, 0x00de, 1}, + {0x0100, 0x0136, 2}, + {0x0139, 0x0147, 2}, + {0x014a, 0x0178, 2}, + {0x0179, 0x017d, 2}, + {0x0181, 0x0182, 1}, + {0x0184, 0x0186, 2}, + {0x0187, 0x0189, 2}, + {0x018a, 0x018b, 1}, + {0x018e, 0x0191, 1}, + {0x0193, 0x0194, 1}, + {0x0196, 0x0198, 1}, + {0x019c, 0x019d, 1}, + {0x019f, 0x01a0, 1}, + {0x01a2, 0x01a6, 2}, + {0x01a7, 0x01a9, 2}, + {0x01ac, 0x01ae, 2}, + {0x01af, 0x01b1, 2}, + {0x01b2, 0x01b3, 1}, + {0x01b5, 0x01b7, 2}, + {0x01b8, 0x01bc, 4}, + {0x01c4, 0x01cd, 3}, + {0x01cf, 0x01db, 2}, + {0x01de, 0x01ee, 2}, + {0x01f1, 0x01f4, 3}, + {0x01f6, 0x01f8, 1}, + {0x01fa, 0x0232, 2}, + {0x023a, 0x023b, 1}, + {0x023d, 0x023e, 1}, + {0x0241, 0x0243, 2}, + {0x0244, 0x0246, 1}, + {0x0248, 0x024e, 2}, + {0x0370, 0x0372, 2}, + {0x0376, 0x0386, 16}, + {0x0388, 0x038a, 1}, + {0x038c, 0x038e, 2}, + {0x038f, 0x0391, 2}, + {0x0392, 0x03a1, 1}, + {0x03a3, 0x03ab, 1}, + {0x03cf, 0x03d2, 3}, + {0x03d3, 0x03d4, 1}, + {0x03d8, 0x03ee, 2}, + {0x03f4, 0x03f7, 3}, + {0x03f9, 0x03fa, 1}, + {0x03fd, 0x042f, 1}, + {0x0460, 0x0480, 2}, + {0x048a, 0x04c0, 2}, + {0x04c1, 0x04cd, 2}, + {0x04d0, 0x0526, 2}, + {0x0531, 0x0556, 1}, + {0x10a0, 0x10c5, 1}, + {0x1e00, 0x1e94, 2}, + {0x1e9e, 0x1efe, 2}, + {0x1f08, 0x1f0f, 1}, + {0x1f18, 0x1f1d, 1}, + {0x1f28, 0x1f2f, 1}, + {0x1f38, 0x1f3f, 1}, + {0x1f48, 0x1f4d, 1}, + {0x1f59, 0x1f5f, 2}, + {0x1f68, 0x1f6f, 1}, + {0x1fb8, 0x1fbb, 1}, + {0x1fc8, 0x1fcb, 1}, + {0x1fd8, 0x1fdb, 1}, + {0x1fe8, 0x1fec, 1}, + {0x1ff8, 0x1ffb, 1}, + {0x2102, 0x2107, 5}, + {0x210b, 0x210d, 1}, + {0x2110, 0x2112, 1}, + {0x2115, 0x2119, 4}, + {0x211a, 0x211d, 1}, + {0x2124, 0x212a, 2}, + {0x212b, 0x212d, 1}, + {0x2130, 0x2133, 1}, + {0x213e, 0x213f, 1}, + {0x2145, 0x2183, 62}, + {0x2c00, 0x2c2e, 1}, + {0x2c60, 0x2c62, 2}, + {0x2c63, 0x2c64, 1}, + {0x2c67, 0x2c6d, 2}, + {0x2c6e, 0x2c70, 1}, + {0x2c72, 0x2c75, 3}, + {0x2c7e, 0x2c80, 1}, + {0x2c82, 0x2ce2, 2}, + {0x2ceb, 0x2ced, 2}, + {0xa640, 0xa66c, 2}, + {0xa680, 0xa696, 2}, + {0xa722, 0xa72e, 2}, + {0xa732, 0xa76e, 2}, + {0xa779, 0xa77d, 2}, + {0xa77e, 0xa786, 2}, + {0xa78b, 0xa78d, 2}, + {0xa790, 0xa7a0, 16}, + {0xa7a2, 0xa7a8, 2}, + {0xff21, 0xff3a, 1}, + }, + R32: []Range32{ + {0x10400, 0x10427, 1}, + {0x1d400, 0x1d419, 1}, + {0x1d434, 0x1d44d, 1}, + {0x1d468, 0x1d481, 1}, + {0x1d49c, 0x1d49e, 2}, + {0x1d49f, 0x1d4a5, 3}, + {0x1d4a6, 0x1d4a9, 3}, + {0x1d4aa, 0x1d4ac, 1}, + {0x1d4ae, 0x1d4b5, 1}, + {0x1d4d0, 0x1d4e9, 1}, + {0x1d504, 0x1d505, 1}, + {0x1d507, 0x1d50a, 1}, + {0x1d50d, 0x1d514, 1}, + {0x1d516, 0x1d51c, 1}, + {0x1d538, 0x1d539, 1}, + {0x1d53b, 0x1d53e, 1}, + {0x1d540, 0x1d544, 1}, + {0x1d546, 0x1d54a, 4}, + {0x1d54b, 0x1d550, 1}, + {0x1d56c, 0x1d585, 1}, + {0x1d5a0, 0x1d5b9, 1}, + {0x1d5d4, 0x1d5ed, 1}, + {0x1d608, 0x1d621, 1}, + {0x1d63c, 0x1d655, 1}, + {0x1d670, 0x1d689, 1}, + {0x1d6a8, 0x1d6c0, 1}, + {0x1d6e2, 0x1d6fa, 1}, + {0x1d71c, 0x1d734, 1}, + {0x1d756, 0x1d76e, 1}, + {0x1d790, 0x1d7a8, 1}, + {0x1d7ca, 0x1d7ca, 1}, + }, +} + +var _Lt = &RangeTable{ + R16: []Range16{ + {0x01c5, 0x01cb, 3}, + {0x01f2, 0x1f88, 7574}, + {0x1f89, 0x1f8f, 1}, + {0x1f98, 0x1f9f, 1}, + {0x1fa8, 0x1faf, 1}, + {0x1fbc, 0x1fcc, 16}, + {0x1ffc, 0x1ffc, 1}, + }, +} + +var _Lo = &RangeTable{ + R16: []Range16{ + {0x01bb, 0x01c0, 5}, + {0x01c1, 0x01c3, 1}, + {0x0294, 0x05d0, 828}, + {0x05d1, 0x05ea, 1}, + {0x05f0, 0x05f2, 1}, + {0x0620, 0x063f, 1}, + {0x0641, 0x064a, 1}, + {0x066e, 0x066f, 1}, + {0x0671, 0x06d3, 1}, + {0x06d5, 0x06ee, 25}, + {0x06ef, 0x06fa, 11}, + {0x06fb, 0x06fc, 1}, + {0x06ff, 0x0710, 17}, + {0x0712, 0x072f, 1}, + {0x074d, 0x07a5, 1}, + {0x07b1, 0x07ca, 25}, + {0x07cb, 0x07ea, 1}, + {0x0800, 0x0815, 1}, + {0x0840, 0x0858, 1}, + {0x0904, 0x0939, 1}, + {0x093d, 0x0950, 19}, + {0x0958, 0x0961, 1}, + {0x0972, 0x0977, 1}, + {0x0979, 0x097f, 1}, + {0x0985, 0x098c, 1}, + {0x098f, 0x0990, 1}, + {0x0993, 0x09a8, 1}, + {0x09aa, 0x09b0, 1}, + {0x09b2, 0x09b6, 4}, + {0x09b7, 0x09b9, 1}, + {0x09bd, 0x09ce, 17}, + {0x09dc, 0x09dd, 1}, + {0x09df, 0x09e1, 1}, + {0x09f0, 0x09f1, 1}, + {0x0a05, 0x0a0a, 1}, + {0x0a0f, 0x0a10, 1}, + {0x0a13, 0x0a28, 1}, + {0x0a2a, 0x0a30, 1}, + {0x0a32, 0x0a33, 1}, + {0x0a35, 0x0a36, 1}, + {0x0a38, 0x0a39, 1}, + {0x0a59, 0x0a5c, 1}, + {0x0a5e, 0x0a72, 20}, + {0x0a73, 0x0a74, 1}, + {0x0a85, 0x0a8d, 1}, + {0x0a8f, 0x0a91, 1}, + {0x0a93, 0x0aa8, 1}, + {0x0aaa, 0x0ab0, 1}, + {0x0ab2, 0x0ab3, 1}, + {0x0ab5, 0x0ab9, 1}, + {0x0abd, 0x0ad0, 19}, + {0x0ae0, 0x0ae1, 1}, + {0x0b05, 0x0b0c, 1}, + {0x0b0f, 0x0b10, 1}, + {0x0b13, 0x0b28, 1}, + {0x0b2a, 0x0b30, 1}, + {0x0b32, 0x0b33, 1}, + {0x0b35, 0x0b39, 1}, + {0x0b3d, 0x0b5c, 31}, + {0x0b5d, 0x0b5f, 2}, + {0x0b60, 0x0b61, 1}, + {0x0b71, 0x0b83, 18}, + {0x0b85, 0x0b8a, 1}, + {0x0b8e, 0x0b90, 1}, + {0x0b92, 0x0b95, 1}, + {0x0b99, 0x0b9a, 1}, + {0x0b9c, 0x0b9e, 2}, + {0x0b9f, 0x0ba3, 4}, + {0x0ba4, 0x0ba8, 4}, + {0x0ba9, 0x0baa, 1}, + {0x0bae, 0x0bb9, 1}, + {0x0bd0, 0x0c05, 53}, + {0x0c06, 0x0c0c, 1}, + {0x0c0e, 0x0c10, 1}, + {0x0c12, 0x0c28, 1}, + {0x0c2a, 0x0c33, 1}, + {0x0c35, 0x0c39, 1}, + {0x0c3d, 0x0c58, 27}, + {0x0c59, 0x0c60, 7}, + {0x0c61, 0x0c85, 36}, + {0x0c86, 0x0c8c, 1}, + {0x0c8e, 0x0c90, 1}, + {0x0c92, 0x0ca8, 1}, + {0x0caa, 0x0cb3, 1}, + {0x0cb5, 0x0cb9, 1}, + {0x0cbd, 0x0cde, 33}, + {0x0ce0, 0x0ce1, 1}, + {0x0cf1, 0x0cf2, 1}, + {0x0d05, 0x0d0c, 1}, + {0x0d0e, 0x0d10, 1}, + {0x0d12, 0x0d3a, 1}, + {0x0d3d, 0x0d4e, 17}, + {0x0d60, 0x0d61, 1}, + {0x0d7a, 0x0d7f, 1}, + {0x0d85, 0x0d96, 1}, + {0x0d9a, 0x0db1, 1}, + {0x0db3, 0x0dbb, 1}, + {0x0dbd, 0x0dc0, 3}, + {0x0dc1, 0x0dc6, 1}, + {0x0e01, 0x0e30, 1}, + {0x0e32, 0x0e33, 1}, + {0x0e40, 0x0e45, 1}, + {0x0e81, 0x0e82, 1}, + {0x0e84, 0x0e87, 3}, + {0x0e88, 0x0e8a, 2}, + {0x0e8d, 0x0e94, 7}, + {0x0e95, 0x0e97, 1}, + {0x0e99, 0x0e9f, 1}, + {0x0ea1, 0x0ea3, 1}, + {0x0ea5, 0x0ea7, 2}, + {0x0eaa, 0x0eab, 1}, + {0x0ead, 0x0eb0, 1}, + {0x0eb2, 0x0eb3, 1}, + {0x0ebd, 0x0ec0, 3}, + {0x0ec1, 0x0ec4, 1}, + {0x0edc, 0x0edd, 1}, + {0x0f00, 0x0f40, 64}, + {0x0f41, 0x0f47, 1}, + {0x0f49, 0x0f6c, 1}, + {0x0f88, 0x0f8c, 1}, + {0x1000, 0x102a, 1}, + {0x103f, 0x1050, 17}, + {0x1051, 0x1055, 1}, + {0x105a, 0x105d, 1}, + {0x1061, 0x1065, 4}, + {0x1066, 0x106e, 8}, + {0x106f, 0x1070, 1}, + {0x1075, 0x1081, 1}, + {0x108e, 0x10d0, 66}, + {0x10d1, 0x10fa, 1}, + {0x1100, 0x1248, 1}, + {0x124a, 0x124d, 1}, + {0x1250, 0x1256, 1}, + {0x1258, 0x125a, 2}, + {0x125b, 0x125d, 1}, + {0x1260, 0x1288, 1}, + {0x128a, 0x128d, 1}, + {0x1290, 0x12b0, 1}, + {0x12b2, 0x12b5, 1}, + {0x12b8, 0x12be, 1}, + {0x12c0, 0x12c2, 2}, + {0x12c3, 0x12c5, 1}, + {0x12c8, 0x12d6, 1}, + {0x12d8, 0x1310, 1}, + {0x1312, 0x1315, 1}, + {0x1318, 0x135a, 1}, + {0x1380, 0x138f, 1}, + {0x13a0, 0x13f4, 1}, + {0x1401, 0x166c, 1}, + {0x166f, 0x167f, 1}, + {0x1681, 0x169a, 1}, + {0x16a0, 0x16ea, 1}, + {0x1700, 0x170c, 1}, + {0x170e, 0x1711, 1}, + {0x1720, 0x1731, 1}, + {0x1740, 0x1751, 1}, + {0x1760, 0x176c, 1}, + {0x176e, 0x1770, 1}, + {0x1780, 0x17b3, 1}, + {0x17dc, 0x1820, 68}, + {0x1821, 0x1842, 1}, + {0x1844, 0x1877, 1}, + {0x1880, 0x18a8, 1}, + {0x18aa, 0x18b0, 6}, + {0x18b1, 0x18f5, 1}, + {0x1900, 0x191c, 1}, + {0x1950, 0x196d, 1}, + {0x1970, 0x1974, 1}, + {0x1980, 0x19ab, 1}, + {0x19c1, 0x19c7, 1}, + {0x1a00, 0x1a16, 1}, + {0x1a20, 0x1a54, 1}, + {0x1b05, 0x1b33, 1}, + {0x1b45, 0x1b4b, 1}, + {0x1b83, 0x1ba0, 1}, + {0x1bae, 0x1baf, 1}, + {0x1bc0, 0x1be5, 1}, + {0x1c00, 0x1c23, 1}, + {0x1c4d, 0x1c4f, 1}, + {0x1c5a, 0x1c77, 1}, + {0x1ce9, 0x1cec, 1}, + {0x1cee, 0x1cf1, 1}, + {0x2135, 0x2138, 1}, + {0x2d30, 0x2d65, 1}, + {0x2d80, 0x2d96, 1}, + {0x2da0, 0x2da6, 1}, + {0x2da8, 0x2dae, 1}, + {0x2db0, 0x2db6, 1}, + {0x2db8, 0x2dbe, 1}, + {0x2dc0, 0x2dc6, 1}, + {0x2dc8, 0x2dce, 1}, + {0x2dd0, 0x2dd6, 1}, + {0x2dd8, 0x2dde, 1}, + {0x3006, 0x303c, 54}, + {0x3041, 0x3096, 1}, + {0x309f, 0x30a1, 2}, + {0x30a2, 0x30fa, 1}, + {0x30ff, 0x3105, 6}, + {0x3106, 0x312d, 1}, + {0x3131, 0x318e, 1}, + {0x31a0, 0x31ba, 1}, + {0x31f0, 0x31ff, 1}, + {0x3400, 0x4db5, 1}, + {0x4e00, 0x9fcb, 1}, + {0xa000, 0xa014, 1}, + {0xa016, 0xa48c, 1}, + {0xa4d0, 0xa4f7, 1}, + {0xa500, 0xa60b, 1}, + {0xa610, 0xa61f, 1}, + {0xa62a, 0xa62b, 1}, + {0xa66e, 0xa6a0, 50}, + {0xa6a1, 0xa6e5, 1}, + {0xa7fb, 0xa801, 1}, + {0xa803, 0xa805, 1}, + {0xa807, 0xa80a, 1}, + {0xa80c, 0xa822, 1}, + {0xa840, 0xa873, 1}, + {0xa882, 0xa8b3, 1}, + {0xa8f2, 0xa8f7, 1}, + {0xa8fb, 0xa90a, 15}, + {0xa90b, 0xa925, 1}, + {0xa930, 0xa946, 1}, + {0xa960, 0xa97c, 1}, + {0xa984, 0xa9b2, 1}, + {0xaa00, 0xaa28, 1}, + {0xaa40, 0xaa42, 1}, + {0xaa44, 0xaa4b, 1}, + {0xaa60, 0xaa6f, 1}, + {0xaa71, 0xaa76, 1}, + {0xaa7a, 0xaa80, 6}, + {0xaa81, 0xaaaf, 1}, + {0xaab1, 0xaab5, 4}, + {0xaab6, 0xaab9, 3}, + {0xaaba, 0xaabd, 1}, + {0xaac0, 0xaac2, 2}, + {0xaadb, 0xaadc, 1}, + {0xab01, 0xab06, 1}, + {0xab09, 0xab0e, 1}, + {0xab11, 0xab16, 1}, + {0xab20, 0xab26, 1}, + {0xab28, 0xab2e, 1}, + {0xabc0, 0xabe2, 1}, + {0xac00, 0xd7a3, 1}, + {0xd7b0, 0xd7c6, 1}, + {0xd7cb, 0xd7fb, 1}, + {0xf900, 0xfa2d, 1}, + {0xfa30, 0xfa6d, 1}, + {0xfa70, 0xfad9, 1}, + {0xfb1d, 0xfb1f, 2}, + {0xfb20, 0xfb28, 1}, + {0xfb2a, 0xfb36, 1}, + {0xfb38, 0xfb3c, 1}, + {0xfb3e, 0xfb40, 2}, + {0xfb41, 0xfb43, 2}, + {0xfb44, 0xfb46, 2}, + {0xfb47, 0xfbb1, 1}, + {0xfbd3, 0xfd3d, 1}, + {0xfd50, 0xfd8f, 1}, + {0xfd92, 0xfdc7, 1}, + {0xfdf0, 0xfdfb, 1}, + {0xfe70, 0xfe74, 1}, + {0xfe76, 0xfefc, 1}, + {0xff66, 0xff6f, 1}, + {0xff71, 0xff9d, 1}, + {0xffa0, 0xffbe, 1}, + {0xffc2, 0xffc7, 1}, + {0xffca, 0xffcf, 1}, + {0xffd2, 0xffd7, 1}, + {0xffda, 0xffdc, 1}, + }, + R32: []Range32{ + {0x10000, 0x1000b, 1}, + {0x1000d, 0x10026, 1}, + {0x10028, 0x1003a, 1}, + {0x1003c, 0x1003d, 1}, + {0x1003f, 0x1004d, 1}, + {0x10050, 0x1005d, 1}, + {0x10080, 0x100fa, 1}, + {0x10280, 0x1029c, 1}, + {0x102a0, 0x102d0, 1}, + {0x10300, 0x1031e, 1}, + {0x10330, 0x10340, 1}, + {0x10342, 0x10349, 1}, + {0x10380, 0x1039d, 1}, + {0x103a0, 0x103c3, 1}, + {0x103c8, 0x103cf, 1}, + {0x10450, 0x1049d, 1}, + {0x10800, 0x10805, 1}, + {0x10808, 0x1080a, 2}, + {0x1080b, 0x10835, 1}, + {0x10837, 0x10838, 1}, + {0x1083c, 0x1083f, 3}, + {0x10840, 0x10855, 1}, + {0x10900, 0x10915, 1}, + {0x10920, 0x10939, 1}, + {0x10a00, 0x10a10, 16}, + {0x10a11, 0x10a13, 1}, + {0x10a15, 0x10a17, 1}, + {0x10a19, 0x10a33, 1}, + {0x10a60, 0x10a7c, 1}, + {0x10b00, 0x10b35, 1}, + {0x10b40, 0x10b55, 1}, + {0x10b60, 0x10b72, 1}, + {0x10c00, 0x10c48, 1}, + {0x11003, 0x11037, 1}, + {0x11083, 0x110af, 1}, + {0x12000, 0x1236e, 1}, + {0x13000, 0x1342e, 1}, + {0x16800, 0x16a38, 1}, + {0x1b000, 0x1b001, 1}, + {0x20000, 0x2a6d6, 1}, + {0x2a700, 0x2b734, 1}, + {0x2b740, 0x2b81d, 1}, + {0x2f800, 0x2fa1d, 1}, + }, } var ( - Cc = _Cc // Cc is the set of Unicode characters in category Cc. - Cf = _Cf // Cf is the set of Unicode characters in category Cf. - Co = _Co // Co is the set of Unicode characters in category Co. - Cs = _Cs // Cs is the set of Unicode characters in category Cs. - Digit = _Nd // Digit is the set of Unicode characters with the "decimal digit" property. - Nd = _Nd // Nd is the set of Unicode characters in category Nd. - Letter = letter // Letter is the set of Unicode letters. - Lm = _Lm // Lm is the set of Unicode characters in category Lm. - Lo = _Lo // Lo is the set of Unicode characters in category Lo. - Lower = _Ll // Lower is the set of Unicode lower case letters. - Ll = _Ll // Ll is the set of Unicode characters in category Ll. - Mc = _Mc // Mc is the set of Unicode characters in category Mc. - Me = _Me // Me is the set of Unicode characters in category Me. - Mn = _Mn // Mn is the set of Unicode characters in category Mn. - Nl = _Nl // Nl is the set of Unicode characters in category Nl. - No = _No // No is the set of Unicode characters in category No. - Pc = _Pc // Pc is the set of Unicode characters in category Pc. - Pd = _Pd // Pd is the set of Unicode characters in category Pd. - Pe = _Pe // Pe is the set of Unicode characters in category Pe. - Pf = _Pf // Pf is the set of Unicode characters in category Pf. - Pi = _Pi // Pi is the set of Unicode characters in category Pi. - Po = _Po // Po is the set of Unicode characters in category Po. - Ps = _Ps // Ps is the set of Unicode characters in category Ps. - Sc = _Sc // Sc is the set of Unicode characters in category Sc. - Sk = _Sk // Sk is the set of Unicode characters in category Sk. - Sm = _Sm // Sm is the set of Unicode characters in category Sm. - So = _So // So is the set of Unicode characters in category So. - Title = _Lt // Title is the set of Unicode title case letters. - Lt = _Lt // Lt is the set of Unicode characters in category Lt. - Upper = _Lu // Upper is the set of Unicode upper case letters. - Lu = _Lu // Lu is the set of Unicode characters in category Lu. - Zl = _Zl // Zl is the set of Unicode characters in category Zl. - Zp = _Zp // Zp is the set of Unicode characters in category Zp. - Zs = _Zs // Zs is the set of Unicode characters in category Zs. + Cc = _Cc // Cc is the set of Unicode characters in category Cc. + Cf = _Cf // Cf is the set of Unicode characters in category Cf. + Co = _Co // Co is the set of Unicode characters in category Co. + Cs = _Cs // Cs is the set of Unicode characters in category Cs. + Digit = _Nd // Digit is the set of Unicode characters with the "decimal digit" property. + Nd = _Nd // Nd is the set of Unicode characters in category Nd. + Letter = _L // Letter/L is the set of Unicode letters, category L. + L = _L + Lm = _Lm // Lm is the set of Unicode characters in category Lm. + Lo = _Lo // Lo is the set of Unicode characters in category Lo. + Lower = _Ll // Lower is the set of Unicode lower case letters. + Ll = _Ll // Ll is the set of Unicode characters in category Ll. + Mark = _M // Mark/M is the set of Unicode mark characters, category M. + M = _M + Mc = _Mc // Mc is the set of Unicode characters in category Mc. + Me = _Me // Me is the set of Unicode characters in category Me. + Mn = _Mn // Mn is the set of Unicode characters in category Mn. + Nl = _Nl // Nl is the set of Unicode characters in category Nl. + No = _No // No is the set of Unicode characters in category No. + Number = _N // Number/N is the set of Unicode number characters, category N. + N = _N + Other = _C // Other/C is the set of Unicode control and special characters, category C. + C = _C + Pc = _Pc // Pc is the set of Unicode characters in category Pc. + Pd = _Pd // Pd is the set of Unicode characters in category Pd. + Pe = _Pe // Pe is the set of Unicode characters in category Pe. + Pf = _Pf // Pf is the set of Unicode characters in category Pf. + Pi = _Pi // Pi is the set of Unicode characters in category Pi. + Po = _Po // Po is the set of Unicode characters in category Po. + Ps = _Ps // Ps is the set of Unicode characters in category Ps. + Punct = _P // Punct/P is the set of Unicode punctuation characters, category P. + P = _P + Sc = _Sc // Sc is the set of Unicode characters in category Sc. + Sk = _Sk // Sk is the set of Unicode characters in category Sk. + Sm = _Sm // Sm is the set of Unicode characters in category Sm. + So = _So // So is the set of Unicode characters in category So. + Space = _Z // Space/Z is the set of Unicode space characters, category Z. + Z = _Z + Symbol = _S // Symbol/S is the set of Unicode symbol characters, category S. + S = _S + Title = _Lt // Title is the set of Unicode title case letters. + Lt = _Lt // Lt is the set of Unicode characters in category Lt. + Upper = _Lu // Upper is the set of Unicode upper case letters. + Lu = _Lu // Lu is the set of Unicode characters in category Lu. + Zl = _Zl // Zl is the set of Unicode characters in category Zl. + Zp = _Zp // Zp is the set of Unicode characters in category Zp. + Zs = _Zs // Zs is the set of Unicode characters in category Zs. ) // Generated by running @@ -2018,7 +2756,7 @@ var ( // DO NOT EDIT // Scripts is the set of Unicode script tables. -var Scripts = map[string][]Range{ +var Scripts = map[string]*RangeTable{ "Katakana": Katakana, "Malayalam": Malayalam, "Phags_Pa": Phags_Pa, @@ -2116,973 +2854,1203 @@ var Scripts = map[string][]Range{ "Gothic": Gothic, } -var _Katakana = []Range{ - {0x30a1, 0x30fa, 1}, - {0x30fd, 0x30ff, 1}, - {0x31f0, 0x31ff, 1}, - {0x32d0, 0x32fe, 1}, - {0x3300, 0x3357, 1}, - {0xff66, 0xff6f, 1}, - {0xff71, 0xff9d, 1}, - {0x1b000, 0x1b000, 1}, -} - -var _Malayalam = []Range{ - {0x0d02, 0x0d03, 1}, - {0x0d05, 0x0d0c, 1}, - {0x0d0e, 0x0d10, 1}, - {0x0d12, 0x0d3a, 1}, - {0x0d3d, 0x0d44, 1}, - {0x0d46, 0x0d48, 1}, - {0x0d4a, 0x0d4e, 1}, - {0x0d57, 0x0d57, 1}, - {0x0d60, 0x0d63, 1}, - {0x0d66, 0x0d75, 1}, - {0x0d79, 0x0d7f, 1}, -} - -var _Phags_Pa = []Range{ - {0xa840, 0xa877, 1}, -} - -var _Inscriptional_Parthian = []Range{ - {0x10b40, 0x10b55, 1}, - {0x10b58, 0x10b5f, 1}, -} - -var _Latin = []Range{ - {0x0041, 0x005a, 1}, - {0x0061, 0x007a, 1}, - {0x00aa, 0x00aa, 1}, - {0x00ba, 0x00ba, 1}, - {0x00c0, 0x00d6, 1}, - {0x00d8, 0x00f6, 1}, - {0x00f8, 0x02b8, 1}, - {0x02e0, 0x02e4, 1}, - {0x1d00, 0x1d25, 1}, - {0x1d2c, 0x1d5c, 1}, - {0x1d62, 0x1d65, 1}, - {0x1d6b, 0x1d77, 1}, - {0x1d79, 0x1dbe, 1}, - {0x1e00, 0x1eff, 1}, - {0x2071, 0x2071, 1}, - {0x207f, 0x207f, 1}, - {0x2090, 0x209c, 1}, - {0x212a, 0x212b, 1}, - {0x2132, 0x2132, 1}, - {0x214e, 0x214e, 1}, - {0x2160, 0x2188, 1}, - {0x2c60, 0x2c7f, 1}, - {0xa722, 0xa787, 1}, - {0xa78b, 0xa78e, 1}, - {0xa790, 0xa791, 1}, - {0xa7a0, 0xa7a9, 1}, - {0xa7fa, 0xa7ff, 1}, - {0xfb00, 0xfb06, 1}, - {0xff21, 0xff3a, 1}, - {0xff41, 0xff5a, 1}, -} - -var _Inscriptional_Pahlavi = []Range{ - {0x10b60, 0x10b72, 1}, - {0x10b78, 0x10b7f, 1}, -} - -var _Osmanya = []Range{ - {0x10480, 0x1049d, 1}, - {0x104a0, 0x104a9, 1}, -} - -var _Khmer = []Range{ - {0x1780, 0x17dd, 1}, - {0x17e0, 0x17e9, 1}, - {0x17f0, 0x17f9, 1}, - {0x19e0, 0x19ff, 1}, -} - -var _Inherited = []Range{ - {0x0300, 0x036f, 1}, - {0x0485, 0x0486, 1}, - {0x064b, 0x0655, 1}, - {0x065f, 0x065f, 1}, - {0x0670, 0x0670, 1}, - {0x0951, 0x0952, 1}, - {0x1cd0, 0x1cd2, 1}, - {0x1cd4, 0x1ce0, 1}, - {0x1ce2, 0x1ce8, 1}, - {0x1ced, 0x1ced, 1}, - {0x1dc0, 0x1de6, 1}, - {0x1dfc, 0x1dff, 1}, - {0x200c, 0x200d, 1}, - {0x20d0, 0x20f0, 1}, - {0x302a, 0x302d, 1}, - {0x3099, 0x309a, 1}, - {0xfe00, 0xfe0f, 1}, - {0xfe20, 0xfe26, 1}, - {0x101fd, 0x101fd, 1}, - {0x1d167, 0x1d169, 1}, - {0x1d17b, 0x1d182, 1}, - {0x1d185, 0x1d18b, 1}, - {0x1d1aa, 0x1d1ad, 1}, - {0xe0100, 0xe01ef, 1}, -} - -var _Telugu = []Range{ - {0x0c01, 0x0c03, 1}, - {0x0c05, 0x0c0c, 1}, - {0x0c0e, 0x0c10, 1}, - {0x0c12, 0x0c28, 1}, - {0x0c2a, 0x0c33, 1}, - {0x0c35, 0x0c39, 1}, - {0x0c3d, 0x0c44, 1}, - {0x0c46, 0x0c48, 1}, - {0x0c4a, 0x0c4d, 1}, - {0x0c55, 0x0c56, 1}, - {0x0c58, 0x0c59, 1}, - {0x0c60, 0x0c63, 1}, - {0x0c66, 0x0c6f, 1}, - {0x0c78, 0x0c7f, 1}, -} - -var _Samaritan = []Range{ - {0x0800, 0x082d, 1}, - {0x0830, 0x083e, 1}, -} - -var _Bopomofo = []Range{ - {0x02ea, 0x02eb, 1}, - {0x3105, 0x312d, 1}, - {0x31a0, 0x31ba, 1}, -} - -var _Imperial_Aramaic = []Range{ - {0x10840, 0x10855, 1}, - {0x10857, 0x1085f, 1}, -} - -var _Kaithi = []Range{ - {0x11080, 0x110c1, 1}, -} - -var _Mandaic = []Range{ - {0x0840, 0x085b, 1}, - {0x085e, 0x085e, 1}, -} - -var _Old_South_Arabian = []Range{ - {0x10a60, 0x10a7f, 1}, -} - -var _Kayah_Li = []Range{ - {0xa900, 0xa92f, 1}, -} - -var _New_Tai_Lue = []Range{ - {0x1980, 0x19ab, 1}, - {0x19b0, 0x19c9, 1}, - {0x19d0, 0x19da, 1}, - {0x19de, 0x19df, 1}, -} - -var _Tai_Le = []Range{ - {0x1950, 0x196d, 1}, - {0x1970, 0x1974, 1}, -} - -var _Kharoshthi = []Range{ - {0x10a00, 0x10a03, 1}, - {0x10a05, 0x10a06, 1}, - {0x10a0c, 0x10a13, 1}, - {0x10a15, 0x10a17, 1}, - {0x10a19, 0x10a33, 1}, - {0x10a38, 0x10a3a, 1}, - {0x10a3f, 0x10a47, 1}, - {0x10a50, 0x10a58, 1}, -} - -var _Common = []Range{ - {0x0000, 0x0040, 1}, - {0x005b, 0x0060, 1}, - {0x007b, 0x00a9, 1}, - {0x00ab, 0x00b9, 1}, - {0x00bb, 0x00bf, 1}, - {0x00d7, 0x00d7, 1}, - {0x00f7, 0x00f7, 1}, - {0x02b9, 0x02df, 1}, - {0x02e5, 0x02e9, 1}, - {0x02ec, 0x02ff, 1}, - {0x0374, 0x0374, 1}, - {0x037e, 0x037e, 1}, - {0x0385, 0x0385, 1}, - {0x0387, 0x0387, 1}, - {0x0589, 0x0589, 1}, - {0x060c, 0x060c, 1}, - {0x061b, 0x061b, 1}, - {0x061f, 0x061f, 1}, - {0x0640, 0x0640, 1}, - {0x0660, 0x0669, 1}, - {0x06dd, 0x06dd, 1}, - {0x0964, 0x0965, 1}, - {0x0970, 0x0970, 1}, - {0x0e3f, 0x0e3f, 1}, - {0x0fd5, 0x0fd8, 1}, - {0x10fb, 0x10fb, 1}, - {0x16eb, 0x16ed, 1}, - {0x1735, 0x1736, 1}, - {0x1802, 0x1803, 1}, - {0x1805, 0x1805, 1}, - {0x1cd3, 0x1cd3, 1}, - {0x1ce1, 0x1ce1, 1}, - {0x1ce9, 0x1cec, 1}, - {0x1cee, 0x1cf2, 1}, - {0x2000, 0x200b, 1}, - {0x200e, 0x2064, 1}, - {0x206a, 0x2070, 1}, - {0x2074, 0x207e, 1}, - {0x2080, 0x208e, 1}, - {0x20a0, 0x20b9, 1}, - {0x2100, 0x2125, 1}, - {0x2127, 0x2129, 1}, - {0x212c, 0x2131, 1}, - {0x2133, 0x214d, 1}, - {0x214f, 0x215f, 1}, - {0x2189, 0x2189, 1}, - {0x2190, 0x23f3, 1}, - {0x2400, 0x2426, 1}, - {0x2440, 0x244a, 1}, - {0x2460, 0x26ff, 1}, - {0x2701, 0x27ca, 1}, - {0x27cc, 0x27cc, 1}, - {0x27ce, 0x27ff, 1}, - {0x2900, 0x2b4c, 1}, - {0x2b50, 0x2b59, 1}, - {0x2e00, 0x2e31, 1}, - {0x2ff0, 0x2ffb, 1}, - {0x3000, 0x3004, 1}, - {0x3006, 0x3006, 1}, - {0x3008, 0x3020, 1}, - {0x3030, 0x3037, 1}, - {0x303c, 0x303f, 1}, - {0x309b, 0x309c, 1}, - {0x30a0, 0x30a0, 1}, - {0x30fb, 0x30fc, 1}, - {0x3190, 0x319f, 1}, - {0x31c0, 0x31e3, 1}, - {0x3220, 0x325f, 1}, - {0x327f, 0x32cf, 1}, - {0x3358, 0x33ff, 1}, - {0x4dc0, 0x4dff, 1}, - {0xa700, 0xa721, 1}, - {0xa788, 0xa78a, 1}, - {0xa830, 0xa839, 1}, - {0xfd3e, 0xfd3f, 1}, - {0xfdfd, 0xfdfd, 1}, - {0xfe10, 0xfe19, 1}, - {0xfe30, 0xfe52, 1}, - {0xfe54, 0xfe66, 1}, - {0xfe68, 0xfe6b, 1}, - {0xfeff, 0xfeff, 1}, - {0xff01, 0xff20, 1}, - {0xff3b, 0xff40, 1}, - {0xff5b, 0xff65, 1}, - {0xff70, 0xff70, 1}, - {0xff9e, 0xff9f, 1}, - {0xffe0, 0xffe6, 1}, - {0xffe8, 0xffee, 1}, - {0xfff9, 0xfffd, 1}, - {0x10100, 0x10102, 1}, - {0x10107, 0x10133, 1}, - {0x10137, 0x1013f, 1}, - {0x10190, 0x1019b, 1}, - {0x101d0, 0x101fc, 1}, - {0x1d000, 0x1d0f5, 1}, - {0x1d100, 0x1d126, 1}, - {0x1d129, 0x1d166, 1}, - {0x1d16a, 0x1d17a, 1}, - {0x1d183, 0x1d184, 1}, - {0x1d18c, 0x1d1a9, 1}, - {0x1d1ae, 0x1d1dd, 1}, - {0x1d300, 0x1d356, 1}, - {0x1d360, 0x1d371, 1}, - {0x1d400, 0x1d454, 1}, - {0x1d456, 0x1d49c, 1}, - {0x1d49e, 0x1d49f, 1}, - {0x1d4a2, 0x1d4a2, 1}, - {0x1d4a5, 0x1d4a6, 1}, - {0x1d4a9, 0x1d4ac, 1}, - {0x1d4ae, 0x1d4b9, 1}, - {0x1d4bb, 0x1d4bb, 1}, - {0x1d4bd, 0x1d4c3, 1}, - {0x1d4c5, 0x1d505, 1}, - {0x1d507, 0x1d50a, 1}, - {0x1d50d, 0x1d514, 1}, - {0x1d516, 0x1d51c, 1}, - {0x1d51e, 0x1d539, 1}, - {0x1d53b, 0x1d53e, 1}, - {0x1d540, 0x1d544, 1}, - {0x1d546, 0x1d546, 1}, - {0x1d54a, 0x1d550, 1}, - {0x1d552, 0x1d6a5, 1}, - {0x1d6a8, 0x1d7cb, 1}, - {0x1d7ce, 0x1d7ff, 1}, - {0x1f000, 0x1f02b, 1}, - {0x1f030, 0x1f093, 1}, - {0x1f0a0, 0x1f0ae, 1}, - {0x1f0b1, 0x1f0be, 1}, - {0x1f0c1, 0x1f0cf, 1}, - {0x1f0d1, 0x1f0df, 1}, - {0x1f100, 0x1f10a, 1}, - {0x1f110, 0x1f12e, 1}, - {0x1f130, 0x1f169, 1}, - {0x1f170, 0x1f19a, 1}, - {0x1f1e6, 0x1f1ff, 1}, - {0x1f201, 0x1f202, 1}, - {0x1f210, 0x1f23a, 1}, - {0x1f240, 0x1f248, 1}, - {0x1f250, 0x1f251, 1}, - {0x1f300, 0x1f320, 1}, - {0x1f330, 0x1f335, 1}, - {0x1f337, 0x1f37c, 1}, - {0x1f380, 0x1f393, 1}, - {0x1f3a0, 0x1f3c4, 1}, - {0x1f3c6, 0x1f3ca, 1}, - {0x1f3e0, 0x1f3f0, 1}, - {0x1f400, 0x1f43e, 1}, - {0x1f440, 0x1f440, 1}, - {0x1f442, 0x1f4f7, 1}, - {0x1f4f9, 0x1f4fc, 1}, - {0x1f500, 0x1f53d, 1}, - {0x1f550, 0x1f567, 1}, - {0x1f5fb, 0x1f5ff, 1}, - {0x1f601, 0x1f610, 1}, - {0x1f612, 0x1f614, 1}, - {0x1f616, 0x1f616, 1}, - {0x1f618, 0x1f618, 1}, - {0x1f61a, 0x1f61a, 1}, - {0x1f61c, 0x1f61e, 1}, - {0x1f620, 0x1f625, 1}, - {0x1f628, 0x1f62b, 1}, - {0x1f62d, 0x1f62d, 1}, - {0x1f630, 0x1f633, 1}, - {0x1f635, 0x1f640, 1}, - {0x1f645, 0x1f64f, 1}, - {0x1f680, 0x1f6c5, 1}, - {0x1f700, 0x1f773, 1}, - {0xe0001, 0xe0001, 1}, - {0xe0020, 0xe007f, 1}, -} - -var _Kannada = []Range{ - {0x0c82, 0x0c83, 1}, - {0x0c85, 0x0c8c, 1}, - {0x0c8e, 0x0c90, 1}, - {0x0c92, 0x0ca8, 1}, - {0x0caa, 0x0cb3, 1}, - {0x0cb5, 0x0cb9, 1}, - {0x0cbc, 0x0cc4, 1}, - {0x0cc6, 0x0cc8, 1}, - {0x0cca, 0x0ccd, 1}, - {0x0cd5, 0x0cd6, 1}, - {0x0cde, 0x0cde, 1}, - {0x0ce0, 0x0ce3, 1}, - {0x0ce6, 0x0cef, 1}, - {0x0cf1, 0x0cf2, 1}, -} - -var _Old_Turkic = []Range{ - {0x10c00, 0x10c48, 1}, -} - -var _Tamil = []Range{ - {0x0b82, 0x0b83, 1}, - {0x0b85, 0x0b8a, 1}, - {0x0b8e, 0x0b90, 1}, - {0x0b92, 0x0b95, 1}, - {0x0b99, 0x0b9a, 1}, - {0x0b9c, 0x0b9c, 1}, - {0x0b9e, 0x0b9f, 1}, - {0x0ba3, 0x0ba4, 1}, - {0x0ba8, 0x0baa, 1}, - {0x0bae, 0x0bb9, 1}, - {0x0bbe, 0x0bc2, 1}, - {0x0bc6, 0x0bc8, 1}, - {0x0bca, 0x0bcd, 1}, - {0x0bd0, 0x0bd0, 1}, - {0x0bd7, 0x0bd7, 1}, - {0x0be6, 0x0bfa, 1}, -} - -var _Tagalog = []Range{ - {0x1700, 0x170c, 1}, - {0x170e, 0x1714, 1}, -} - -var _Brahmi = []Range{ - {0x11000, 0x1104d, 1}, - {0x11052, 0x1106f, 1}, -} - -var _Arabic = []Range{ - {0x0600, 0x0603, 1}, - {0x0606, 0x060b, 1}, - {0x060d, 0x061a, 1}, - {0x061e, 0x061e, 1}, - {0x0620, 0x063f, 1}, - {0x0641, 0x064a, 1}, - {0x0656, 0x065e, 1}, - {0x066a, 0x066f, 1}, - {0x0671, 0x06dc, 1}, - {0x06de, 0x06ff, 1}, - {0x0750, 0x077f, 1}, - {0xfb50, 0xfbc1, 1}, - {0xfbd3, 0xfd3d, 1}, - {0xfd50, 0xfd8f, 1}, - {0xfd92, 0xfdc7, 1}, - {0xfdf0, 0xfdfc, 1}, - {0xfe70, 0xfe74, 1}, - {0xfe76, 0xfefc, 1}, - {0x10e60, 0x10e7e, 1}, -} - -var _Tagbanwa = []Range{ - {0x1760, 0x176c, 1}, - {0x176e, 0x1770, 1}, - {0x1772, 0x1773, 1}, -} - -var _Canadian_Aboriginal = []Range{ - {0x1400, 0x167f, 1}, - {0x18b0, 0x18f5, 1}, -} - -var _Tibetan = []Range{ - {0x0f00, 0x0f47, 1}, - {0x0f49, 0x0f6c, 1}, - {0x0f71, 0x0f97, 1}, - {0x0f99, 0x0fbc, 1}, - {0x0fbe, 0x0fcc, 1}, - {0x0fce, 0x0fd4, 1}, - {0x0fd9, 0x0fda, 1}, -} - -var _Coptic = []Range{ - {0x03e2, 0x03ef, 1}, - {0x2c80, 0x2cf1, 1}, - {0x2cf9, 0x2cff, 1}, -} - -var _Hiragana = []Range{ - {0x3041, 0x3096, 1}, - {0x309d, 0x309f, 1}, - {0x1b001, 0x1b001, 1}, - {0x1f200, 0x1f200, 1}, -} - -var _Limbu = []Range{ - {0x1900, 0x191c, 1}, - {0x1920, 0x192b, 1}, - {0x1930, 0x193b, 1}, - {0x1940, 0x1940, 1}, - {0x1944, 0x194f, 1}, -} - -var _Egyptian_Hieroglyphs = []Range{ - {0x13000, 0x1342e, 1}, -} - -var _Avestan = []Range{ - {0x10b00, 0x10b35, 1}, - {0x10b39, 0x10b3f, 1}, -} - -var _Myanmar = []Range{ - {0x1000, 0x109f, 1}, - {0xaa60, 0xaa7b, 1}, -} - -var _Armenian = []Range{ - {0x0531, 0x0556, 1}, - {0x0559, 0x055f, 1}, - {0x0561, 0x0587, 1}, - {0x058a, 0x058a, 1}, - {0xfb13, 0xfb17, 1}, -} - -var _Sinhala = []Range{ - {0x0d82, 0x0d83, 1}, - {0x0d85, 0x0d96, 1}, - {0x0d9a, 0x0db1, 1}, - {0x0db3, 0x0dbb, 1}, - {0x0dbd, 0x0dbd, 1}, - {0x0dc0, 0x0dc6, 1}, - {0x0dca, 0x0dca, 1}, - {0x0dcf, 0x0dd4, 1}, - {0x0dd6, 0x0dd6, 1}, - {0x0dd8, 0x0ddf, 1}, - {0x0df2, 0x0df4, 1}, -} - -var _Bengali = []Range{ - {0x0981, 0x0983, 1}, - {0x0985, 0x098c, 1}, - {0x098f, 0x0990, 1}, - {0x0993, 0x09a8, 1}, - {0x09aa, 0x09b0, 1}, - {0x09b2, 0x09b2, 1}, - {0x09b6, 0x09b9, 1}, - {0x09bc, 0x09c4, 1}, - {0x09c7, 0x09c8, 1}, - {0x09cb, 0x09ce, 1}, - {0x09d7, 0x09d7, 1}, - {0x09dc, 0x09dd, 1}, - {0x09df, 0x09e3, 1}, - {0x09e6, 0x09fb, 1}, -} - -var _Greek = []Range{ - {0x0370, 0x0373, 1}, - {0x0375, 0x0377, 1}, - {0x037a, 0x037d, 1}, - {0x0384, 0x0384, 1}, - {0x0386, 0x0386, 1}, - {0x0388, 0x038a, 1}, - {0x038c, 0x038c, 1}, - {0x038e, 0x03a1, 1}, - {0x03a3, 0x03e1, 1}, - {0x03f0, 0x03ff, 1}, - {0x1d26, 0x1d2a, 1}, - {0x1d5d, 0x1d61, 1}, - {0x1d66, 0x1d6a, 1}, - {0x1dbf, 0x1dbf, 1}, - {0x1f00, 0x1f15, 1}, - {0x1f18, 0x1f1d, 1}, - {0x1f20, 0x1f45, 1}, - {0x1f48, 0x1f4d, 1}, - {0x1f50, 0x1f57, 1}, - {0x1f59, 0x1f59, 1}, - {0x1f5b, 0x1f5b, 1}, - {0x1f5d, 0x1f5d, 1}, - {0x1f5f, 0x1f7d, 1}, - {0x1f80, 0x1fb4, 1}, - {0x1fb6, 0x1fc4, 1}, - {0x1fc6, 0x1fd3, 1}, - {0x1fd6, 0x1fdb, 1}, - {0x1fdd, 0x1fef, 1}, - {0x1ff2, 0x1ff4, 1}, - {0x1ff6, 0x1ffe, 1}, - {0x2126, 0x2126, 1}, - {0x10140, 0x1018a, 1}, - {0x1d200, 0x1d245, 1}, -} - -var _Cham = []Range{ - {0xaa00, 0xaa36, 1}, - {0xaa40, 0xaa4d, 1}, - {0xaa50, 0xaa59, 1}, - {0xaa5c, 0xaa5f, 1}, -} - -var _Hebrew = []Range{ - {0x0591, 0x05c7, 1}, - {0x05d0, 0x05ea, 1}, - {0x05f0, 0x05f4, 1}, - {0xfb1d, 0xfb36, 1}, - {0xfb38, 0xfb3c, 1}, - {0xfb3e, 0xfb3e, 1}, - {0xfb40, 0xfb41, 1}, - {0xfb43, 0xfb44, 1}, - {0xfb46, 0xfb4f, 1}, -} - -var _Meetei_Mayek = []Range{ - {0xabc0, 0xabed, 1}, - {0xabf0, 0xabf9, 1}, -} - -var _Saurashtra = []Range{ - {0xa880, 0xa8c4, 1}, - {0xa8ce, 0xa8d9, 1}, -} - -var _Hangul = []Range{ - {0x1100, 0x11ff, 1}, - {0x302e, 0x302f, 1}, - {0x3131, 0x318e, 1}, - {0x3200, 0x321e, 1}, - {0x3260, 0x327e, 1}, - {0xa960, 0xa97c, 1}, - {0xac00, 0xd7a3, 1}, - {0xd7b0, 0xd7c6, 1}, - {0xd7cb, 0xd7fb, 1}, - {0xffa0, 0xffbe, 1}, - {0xffc2, 0xffc7, 1}, - {0xffca, 0xffcf, 1}, - {0xffd2, 0xffd7, 1}, - {0xffda, 0xffdc, 1}, -} - -var _Runic = []Range{ - {0x16a0, 0x16ea, 1}, - {0x16ee, 0x16f0, 1}, -} - -var _Deseret = []Range{ - {0x10400, 0x1044f, 1}, -} - -var _Lisu = []Range{ - {0xa4d0, 0xa4ff, 1}, -} - -var _Sundanese = []Range{ - {0x1b80, 0x1baa, 1}, - {0x1bae, 0x1bb9, 1}, -} - -var _Glagolitic = []Range{ - {0x2c00, 0x2c2e, 1}, - {0x2c30, 0x2c5e, 1}, -} - -var _Oriya = []Range{ - {0x0b01, 0x0b03, 1}, - {0x0b05, 0x0b0c, 1}, - {0x0b0f, 0x0b10, 1}, - {0x0b13, 0x0b28, 1}, - {0x0b2a, 0x0b30, 1}, - {0x0b32, 0x0b33, 1}, - {0x0b35, 0x0b39, 1}, - {0x0b3c, 0x0b44, 1}, - {0x0b47, 0x0b48, 1}, - {0x0b4b, 0x0b4d, 1}, - {0x0b56, 0x0b57, 1}, - {0x0b5c, 0x0b5d, 1}, - {0x0b5f, 0x0b63, 1}, - {0x0b66, 0x0b77, 1}, -} - -var _Buhid = []Range{ - {0x1740, 0x1753, 1}, -} - -var _Ethiopic = []Range{ - {0x1200, 0x1248, 1}, - {0x124a, 0x124d, 1}, - {0x1250, 0x1256, 1}, - {0x1258, 0x1258, 1}, - {0x125a, 0x125d, 1}, - {0x1260, 0x1288, 1}, - {0x128a, 0x128d, 1}, - {0x1290, 0x12b0, 1}, - {0x12b2, 0x12b5, 1}, - {0x12b8, 0x12be, 1}, - {0x12c0, 0x12c0, 1}, - {0x12c2, 0x12c5, 1}, - {0x12c8, 0x12d6, 1}, - {0x12d8, 0x1310, 1}, - {0x1312, 0x1315, 1}, - {0x1318, 0x135a, 1}, - {0x135d, 0x137c, 1}, - {0x1380, 0x1399, 1}, - {0x2d80, 0x2d96, 1}, - {0x2da0, 0x2da6, 1}, - {0x2da8, 0x2dae, 1}, - {0x2db0, 0x2db6, 1}, - {0x2db8, 0x2dbe, 1}, - {0x2dc0, 0x2dc6, 1}, - {0x2dc8, 0x2dce, 1}, - {0x2dd0, 0x2dd6, 1}, - {0x2dd8, 0x2dde, 1}, - {0xab01, 0xab06, 1}, - {0xab09, 0xab0e, 1}, - {0xab11, 0xab16, 1}, - {0xab20, 0xab26, 1}, - {0xab28, 0xab2e, 1}, -} - -var _Javanese = []Range{ - {0xa980, 0xa9cd, 1}, - {0xa9cf, 0xa9d9, 1}, - {0xa9de, 0xa9df, 1}, -} - -var _Syloti_Nagri = []Range{ - {0xa800, 0xa82b, 1}, -} - -var _Vai = []Range{ - {0xa500, 0xa62b, 1}, -} - -var _Cherokee = []Range{ - {0x13a0, 0x13f4, 1}, -} - -var _Ogham = []Range{ - {0x1680, 0x169c, 1}, -} - -var _Batak = []Range{ - {0x1bc0, 0x1bf3, 1}, - {0x1bfc, 0x1bff, 1}, -} - -var _Syriac = []Range{ - {0x0700, 0x070d, 1}, - {0x070f, 0x074a, 1}, - {0x074d, 0x074f, 1}, -} - -var _Gurmukhi = []Range{ - {0x0a01, 0x0a03, 1}, - {0x0a05, 0x0a0a, 1}, - {0x0a0f, 0x0a10, 1}, - {0x0a13, 0x0a28, 1}, - {0x0a2a, 0x0a30, 1}, - {0x0a32, 0x0a33, 1}, - {0x0a35, 0x0a36, 1}, - {0x0a38, 0x0a39, 1}, - {0x0a3c, 0x0a3c, 1}, - {0x0a3e, 0x0a42, 1}, - {0x0a47, 0x0a48, 1}, - {0x0a4b, 0x0a4d, 1}, - {0x0a51, 0x0a51, 1}, - {0x0a59, 0x0a5c, 1}, - {0x0a5e, 0x0a5e, 1}, - {0x0a66, 0x0a75, 1}, -} - -var _Tai_Tham = []Range{ - {0x1a20, 0x1a5e, 1}, - {0x1a60, 0x1a7c, 1}, - {0x1a7f, 0x1a89, 1}, - {0x1a90, 0x1a99, 1}, - {0x1aa0, 0x1aad, 1}, -} - -var _Ol_Chiki = []Range{ - {0x1c50, 0x1c7f, 1}, -} - -var _Mongolian = []Range{ - {0x1800, 0x1801, 1}, - {0x1804, 0x1804, 1}, - {0x1806, 0x180e, 1}, - {0x1810, 0x1819, 1}, - {0x1820, 0x1877, 1}, - {0x1880, 0x18aa, 1}, -} - -var _Hanunoo = []Range{ - {0x1720, 0x1734, 1}, -} - -var _Cypriot = []Range{ - {0x10800, 0x10805, 1}, - {0x10808, 0x10808, 1}, - {0x1080a, 0x10835, 1}, - {0x10837, 0x10838, 1}, - {0x1083c, 0x1083c, 1}, - {0x1083f, 0x1083f, 1}, -} - -var _Buginese = []Range{ - {0x1a00, 0x1a1b, 1}, - {0x1a1e, 0x1a1f, 1}, -} - -var _Bamum = []Range{ - {0xa6a0, 0xa6f7, 1}, - {0x16800, 0x16a38, 1}, -} - -var _Lepcha = []Range{ - {0x1c00, 0x1c37, 1}, - {0x1c3b, 0x1c49, 1}, - {0x1c4d, 0x1c4f, 1}, -} - -var _Thaana = []Range{ - {0x0780, 0x07b1, 1}, -} - -var _Old_Persian = []Range{ - {0x103a0, 0x103c3, 1}, - {0x103c8, 0x103d5, 1}, -} - -var _Cuneiform = []Range{ - {0x12000, 0x1236e, 1}, - {0x12400, 0x12462, 1}, - {0x12470, 0x12473, 1}, -} - -var _Rejang = []Range{ - {0xa930, 0xa953, 1}, - {0xa95f, 0xa95f, 1}, -} - -var _Georgian = []Range{ - {0x10a0, 0x10c5, 1}, - {0x10d0, 0x10fa, 1}, - {0x10fc, 0x10fc, 1}, - {0x2d00, 0x2d25, 1}, -} - -var _Shavian = []Range{ - {0x10450, 0x1047f, 1}, -} - -var _Lycian = []Range{ - {0x10280, 0x1029c, 1}, -} - -var _Nko = []Range{ - {0x07c0, 0x07fa, 1}, -} - -var _Yi = []Range{ - {0xa000, 0xa48c, 1}, - {0xa490, 0xa4c6, 1}, -} - -var _Lao = []Range{ - {0x0e81, 0x0e82, 1}, - {0x0e84, 0x0e84, 1}, - {0x0e87, 0x0e88, 1}, - {0x0e8a, 0x0e8a, 1}, - {0x0e8d, 0x0e8d, 1}, - {0x0e94, 0x0e97, 1}, - {0x0e99, 0x0e9f, 1}, - {0x0ea1, 0x0ea3, 1}, - {0x0ea5, 0x0ea5, 1}, - {0x0ea7, 0x0ea7, 1}, - {0x0eaa, 0x0eab, 1}, - {0x0ead, 0x0eb9, 1}, - {0x0ebb, 0x0ebd, 1}, - {0x0ec0, 0x0ec4, 1}, - {0x0ec6, 0x0ec6, 1}, - {0x0ec8, 0x0ecd, 1}, - {0x0ed0, 0x0ed9, 1}, - {0x0edc, 0x0edd, 1}, -} - -var _Linear_B = []Range{ - {0x10000, 0x1000b, 1}, - {0x1000d, 0x10026, 1}, - {0x10028, 0x1003a, 1}, - {0x1003c, 0x1003d, 1}, - {0x1003f, 0x1004d, 1}, - {0x10050, 0x1005d, 1}, - {0x10080, 0x100fa, 1}, -} - -var _Old_Italic = []Range{ - {0x10300, 0x1031e, 1}, - {0x10320, 0x10323, 1}, -} - -var _Tai_Viet = []Range{ - {0xaa80, 0xaac2, 1}, - {0xaadb, 0xaadf, 1}, -} - -var _Devanagari = []Range{ - {0x0900, 0x0950, 1}, - {0x0953, 0x0963, 1}, - {0x0966, 0x096f, 1}, - {0x0971, 0x0977, 1}, - {0x0979, 0x097f, 1}, - {0xa8e0, 0xa8fb, 1}, -} - -var _Lydian = []Range{ - {0x10920, 0x10939, 1}, - {0x1093f, 0x1093f, 1}, -} - -var _Tifinagh = []Range{ - {0x2d30, 0x2d65, 1}, - {0x2d6f, 0x2d70, 1}, - {0x2d7f, 0x2d7f, 1}, -} - -var _Ugaritic = []Range{ - {0x10380, 0x1039d, 1}, - {0x1039f, 0x1039f, 1}, -} - -var _Thai = []Range{ - {0x0e01, 0x0e3a, 1}, - {0x0e40, 0x0e5b, 1}, -} - -var _Cyrillic = []Range{ - {0x0400, 0x0484, 1}, - {0x0487, 0x0527, 1}, - {0x1d2b, 0x1d2b, 1}, - {0x1d78, 0x1d78, 1}, - {0x2de0, 0x2dff, 1}, - {0xa640, 0xa673, 1}, - {0xa67c, 0xa697, 1}, -} - -var _Gujarati = []Range{ - {0x0a81, 0x0a83, 1}, - {0x0a85, 0x0a8d, 1}, - {0x0a8f, 0x0a91, 1}, - {0x0a93, 0x0aa8, 1}, - {0x0aaa, 0x0ab0, 1}, - {0x0ab2, 0x0ab3, 1}, - {0x0ab5, 0x0ab9, 1}, - {0x0abc, 0x0ac5, 1}, - {0x0ac7, 0x0ac9, 1}, - {0x0acb, 0x0acd, 1}, - {0x0ad0, 0x0ad0, 1}, - {0x0ae0, 0x0ae3, 1}, - {0x0ae6, 0x0aef, 1}, - {0x0af1, 0x0af1, 1}, -} - -var _Carian = []Range{ - {0x102a0, 0x102d0, 1}, -} - -var _Phoenician = []Range{ - {0x10900, 0x1091b, 1}, - {0x1091f, 0x1091f, 1}, -} - -var _Balinese = []Range{ - {0x1b00, 0x1b4b, 1}, - {0x1b50, 0x1b7c, 1}, -} - -var _Braille = []Range{ - {0x2800, 0x28ff, 1}, -} - -var _Han = []Range{ - {0x2e80, 0x2e99, 1}, - {0x2e9b, 0x2ef3, 1}, - {0x2f00, 0x2fd5, 1}, - {0x3005, 0x3005, 1}, - {0x3007, 0x3007, 1}, - {0x3021, 0x3029, 1}, - {0x3038, 0x303b, 1}, - {0x3400, 0x4db5, 1}, - {0x4e00, 0x9fcb, 1}, - {0xf900, 0xfa2d, 1}, - {0xfa30, 0xfa6d, 1}, - {0xfa70, 0xfad9, 1}, - {0x20000, 0x2a6d6, 1}, - {0x2a700, 0x2b734, 1}, - {0x2b740, 0x2b81d, 1}, - {0x2f800, 0x2fa1d, 1}, -} - -var _Gothic = []Range{ - {0x10330, 0x1034a, 1}, +var _Katakana = &RangeTable{ + R16: []Range16{ + {0x30a1, 0x30fa, 1}, + {0x30fd, 0x30ff, 1}, + {0x31f0, 0x31ff, 1}, + {0x32d0, 0x32fe, 1}, + {0x3300, 0x3357, 1}, + {0xff66, 0xff6f, 1}, + {0xff71, 0xff9d, 1}, + }, + R32: []Range32{ + {0x1b000, 0x1b000, 1}, + }, +} + +var _Malayalam = &RangeTable{ + R16: []Range16{ + {0x0d02, 0x0d03, 1}, + {0x0d05, 0x0d0c, 1}, + {0x0d0e, 0x0d10, 1}, + {0x0d12, 0x0d3a, 1}, + {0x0d3d, 0x0d44, 1}, + {0x0d46, 0x0d48, 1}, + {0x0d4a, 0x0d4e, 1}, + {0x0d57, 0x0d57, 1}, + {0x0d60, 0x0d63, 1}, + {0x0d66, 0x0d75, 1}, + {0x0d79, 0x0d7f, 1}, + }, +} + +var _Phags_Pa = &RangeTable{ + R16: []Range16{ + {0xa840, 0xa877, 1}, + }, +} + +var _Inscriptional_Parthian = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10b40, 0x10b55, 1}, + {0x10b58, 0x10b5f, 1}, + }, +} + +var _Latin = &RangeTable{ + R16: []Range16{ + {0x0041, 0x005a, 1}, + {0x0061, 0x007a, 1}, + {0x00aa, 0x00aa, 1}, + {0x00ba, 0x00ba, 1}, + {0x00c0, 0x00d6, 1}, + {0x00d8, 0x00f6, 1}, + {0x00f8, 0x02b8, 1}, + {0x02e0, 0x02e4, 1}, + {0x1d00, 0x1d25, 1}, + {0x1d2c, 0x1d5c, 1}, + {0x1d62, 0x1d65, 1}, + {0x1d6b, 0x1d77, 1}, + {0x1d79, 0x1dbe, 1}, + {0x1e00, 0x1eff, 1}, + {0x2071, 0x2071, 1}, + {0x207f, 0x207f, 1}, + {0x2090, 0x209c, 1}, + {0x212a, 0x212b, 1}, + {0x2132, 0x2132, 1}, + {0x214e, 0x214e, 1}, + {0x2160, 0x2188, 1}, + {0x2c60, 0x2c7f, 1}, + {0xa722, 0xa787, 1}, + {0xa78b, 0xa78e, 1}, + {0xa790, 0xa791, 1}, + {0xa7a0, 0xa7a9, 1}, + {0xa7fa, 0xa7ff, 1}, + {0xfb00, 0xfb06, 1}, + {0xff21, 0xff3a, 1}, + {0xff41, 0xff5a, 1}, + }, +} + +var _Inscriptional_Pahlavi = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10b60, 0x10b72, 1}, + {0x10b78, 0x10b7f, 1}, + }, +} + +var _Osmanya = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10480, 0x1049d, 1}, + {0x104a0, 0x104a9, 1}, + }, +} + +var _Khmer = &RangeTable{ + R16: []Range16{ + {0x1780, 0x17dd, 1}, + {0x17e0, 0x17e9, 1}, + {0x17f0, 0x17f9, 1}, + {0x19e0, 0x19ff, 1}, + }, +} + +var _Inherited = &RangeTable{ + R16: []Range16{ + {0x0300, 0x036f, 1}, + {0x0485, 0x0486, 1}, + {0x064b, 0x0655, 1}, + {0x065f, 0x065f, 1}, + {0x0670, 0x0670, 1}, + {0x0951, 0x0952, 1}, + {0x1cd0, 0x1cd2, 1}, + {0x1cd4, 0x1ce0, 1}, + {0x1ce2, 0x1ce8, 1}, + {0x1ced, 0x1ced, 1}, + {0x1dc0, 0x1de6, 1}, + {0x1dfc, 0x1dff, 1}, + {0x200c, 0x200d, 1}, + {0x20d0, 0x20f0, 1}, + {0x302a, 0x302d, 1}, + {0x3099, 0x309a, 1}, + {0xfe00, 0xfe0f, 1}, + {0xfe20, 0xfe26, 1}, + }, + R32: []Range32{ + {0x101fd, 0x101fd, 1}, + {0x1d167, 0x1d169, 1}, + {0x1d17b, 0x1d182, 1}, + {0x1d185, 0x1d18b, 1}, + {0x1d1aa, 0x1d1ad, 1}, + {0xe0100, 0xe01ef, 1}, + }, +} + +var _Telugu = &RangeTable{ + R16: []Range16{ + {0x0c01, 0x0c03, 1}, + {0x0c05, 0x0c0c, 1}, + {0x0c0e, 0x0c10, 1}, + {0x0c12, 0x0c28, 1}, + {0x0c2a, 0x0c33, 1}, + {0x0c35, 0x0c39, 1}, + {0x0c3d, 0x0c44, 1}, + {0x0c46, 0x0c48, 1}, + {0x0c4a, 0x0c4d, 1}, + {0x0c55, 0x0c56, 1}, + {0x0c58, 0x0c59, 1}, + {0x0c60, 0x0c63, 1}, + {0x0c66, 0x0c6f, 1}, + {0x0c78, 0x0c7f, 1}, + }, +} + +var _Samaritan = &RangeTable{ + R16: []Range16{ + {0x0800, 0x082d, 1}, + {0x0830, 0x083e, 1}, + }, +} + +var _Bopomofo = &RangeTable{ + R16: []Range16{ + {0x02ea, 0x02eb, 1}, + {0x3105, 0x312d, 1}, + {0x31a0, 0x31ba, 1}, + }, +} + +var _Imperial_Aramaic = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10840, 0x10855, 1}, + {0x10857, 0x1085f, 1}, + }, +} + +var _Kaithi = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x11080, 0x110c1, 1}, + }, +} + +var _Mandaic = &RangeTable{ + R16: []Range16{ + {0x0840, 0x085b, 1}, + {0x085e, 0x085e, 1}, + }, +} + +var _Old_South_Arabian = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10a60, 0x10a7f, 1}, + }, +} + +var _Kayah_Li = &RangeTable{ + R16: []Range16{ + {0xa900, 0xa92f, 1}, + }, +} + +var _New_Tai_Lue = &RangeTable{ + R16: []Range16{ + {0x1980, 0x19ab, 1}, + {0x19b0, 0x19c9, 1}, + {0x19d0, 0x19da, 1}, + {0x19de, 0x19df, 1}, + }, +} + +var _Tai_Le = &RangeTable{ + R16: []Range16{ + {0x1950, 0x196d, 1}, + {0x1970, 0x1974, 1}, + }, +} + +var _Kharoshthi = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10a00, 0x10a03, 1}, + {0x10a05, 0x10a06, 1}, + {0x10a0c, 0x10a13, 1}, + {0x10a15, 0x10a17, 1}, + {0x10a19, 0x10a33, 1}, + {0x10a38, 0x10a3a, 1}, + {0x10a3f, 0x10a47, 1}, + {0x10a50, 0x10a58, 1}, + }, +} + +var _Common = &RangeTable{ + R16: []Range16{ + {0x0000, 0x0040, 1}, + {0x005b, 0x0060, 1}, + {0x007b, 0x00a9, 1}, + {0x00ab, 0x00b9, 1}, + {0x00bb, 0x00bf, 1}, + {0x00d7, 0x00d7, 1}, + {0x00f7, 0x00f7, 1}, + {0x02b9, 0x02df, 1}, + {0x02e5, 0x02e9, 1}, + {0x02ec, 0x02ff, 1}, + {0x0374, 0x0374, 1}, + {0x037e, 0x037e, 1}, + {0x0385, 0x0385, 1}, + {0x0387, 0x0387, 1}, + {0x0589, 0x0589, 1}, + {0x060c, 0x060c, 1}, + {0x061b, 0x061b, 1}, + {0x061f, 0x061f, 1}, + {0x0640, 0x0640, 1}, + {0x0660, 0x0669, 1}, + {0x06dd, 0x06dd, 1}, + {0x0964, 0x0965, 1}, + {0x0970, 0x0970, 1}, + {0x0e3f, 0x0e3f, 1}, + {0x0fd5, 0x0fd8, 1}, + {0x10fb, 0x10fb, 1}, + {0x16eb, 0x16ed, 1}, + {0x1735, 0x1736, 1}, + {0x1802, 0x1803, 1}, + {0x1805, 0x1805, 1}, + {0x1cd3, 0x1cd3, 1}, + {0x1ce1, 0x1ce1, 1}, + {0x1ce9, 0x1cec, 1}, + {0x1cee, 0x1cf2, 1}, + {0x2000, 0x200b, 1}, + {0x200e, 0x2064, 1}, + {0x206a, 0x2070, 1}, + {0x2074, 0x207e, 1}, + {0x2080, 0x208e, 1}, + {0x20a0, 0x20b9, 1}, + {0x2100, 0x2125, 1}, + {0x2127, 0x2129, 1}, + {0x212c, 0x2131, 1}, + {0x2133, 0x214d, 1}, + {0x214f, 0x215f, 1}, + {0x2189, 0x2189, 1}, + {0x2190, 0x23f3, 1}, + {0x2400, 0x2426, 1}, + {0x2440, 0x244a, 1}, + {0x2460, 0x26ff, 1}, + {0x2701, 0x27ca, 1}, + {0x27cc, 0x27cc, 1}, + {0x27ce, 0x27ff, 1}, + {0x2900, 0x2b4c, 1}, + {0x2b50, 0x2b59, 1}, + {0x2e00, 0x2e31, 1}, + {0x2ff0, 0x2ffb, 1}, + {0x3000, 0x3004, 1}, + {0x3006, 0x3006, 1}, + {0x3008, 0x3020, 1}, + {0x3030, 0x3037, 1}, + {0x303c, 0x303f, 1}, + {0x309b, 0x309c, 1}, + {0x30a0, 0x30a0, 1}, + {0x30fb, 0x30fc, 1}, + {0x3190, 0x319f, 1}, + {0x31c0, 0x31e3, 1}, + {0x3220, 0x325f, 1}, + {0x327f, 0x32cf, 1}, + {0x3358, 0x33ff, 1}, + {0x4dc0, 0x4dff, 1}, + {0xa700, 0xa721, 1}, + {0xa788, 0xa78a, 1}, + {0xa830, 0xa839, 1}, + {0xfd3e, 0xfd3f, 1}, + {0xfdfd, 0xfdfd, 1}, + {0xfe10, 0xfe19, 1}, + {0xfe30, 0xfe52, 1}, + {0xfe54, 0xfe66, 1}, + {0xfe68, 0xfe6b, 1}, + {0xfeff, 0xfeff, 1}, + {0xff01, 0xff20, 1}, + {0xff3b, 0xff40, 1}, + {0xff5b, 0xff65, 1}, + {0xff70, 0xff70, 1}, + {0xff9e, 0xff9f, 1}, + {0xffe0, 0xffe6, 1}, + {0xffe8, 0xffee, 1}, + {0xfff9, 0xfffd, 1}, + }, + R32: []Range32{ + {0x10100, 0x10102, 1}, + {0x10107, 0x10133, 1}, + {0x10137, 0x1013f, 1}, + {0x10190, 0x1019b, 1}, + {0x101d0, 0x101fc, 1}, + {0x1d000, 0x1d0f5, 1}, + {0x1d100, 0x1d126, 1}, + {0x1d129, 0x1d166, 1}, + {0x1d16a, 0x1d17a, 1}, + {0x1d183, 0x1d184, 1}, + {0x1d18c, 0x1d1a9, 1}, + {0x1d1ae, 0x1d1dd, 1}, + {0x1d300, 0x1d356, 1}, + {0x1d360, 0x1d371, 1}, + {0x1d400, 0x1d454, 1}, + {0x1d456, 0x1d49c, 1}, + {0x1d49e, 0x1d49f, 1}, + {0x1d4a2, 0x1d4a2, 1}, + {0x1d4a5, 0x1d4a6, 1}, + {0x1d4a9, 0x1d4ac, 1}, + {0x1d4ae, 0x1d4b9, 1}, + {0x1d4bb, 0x1d4bb, 1}, + {0x1d4bd, 0x1d4c3, 1}, + {0x1d4c5, 0x1d505, 1}, + {0x1d507, 0x1d50a, 1}, + {0x1d50d, 0x1d514, 1}, + {0x1d516, 0x1d51c, 1}, + {0x1d51e, 0x1d539, 1}, + {0x1d53b, 0x1d53e, 1}, + {0x1d540, 0x1d544, 1}, + {0x1d546, 0x1d546, 1}, + {0x1d54a, 0x1d550, 1}, + {0x1d552, 0x1d6a5, 1}, + {0x1d6a8, 0x1d7cb, 1}, + {0x1d7ce, 0x1d7ff, 1}, + {0x1f000, 0x1f02b, 1}, + {0x1f030, 0x1f093, 1}, + {0x1f0a0, 0x1f0ae, 1}, + {0x1f0b1, 0x1f0be, 1}, + {0x1f0c1, 0x1f0cf, 1}, + {0x1f0d1, 0x1f0df, 1}, + {0x1f100, 0x1f10a, 1}, + {0x1f110, 0x1f12e, 1}, + {0x1f130, 0x1f169, 1}, + {0x1f170, 0x1f19a, 1}, + {0x1f1e6, 0x1f1ff, 1}, + {0x1f201, 0x1f202, 1}, + {0x1f210, 0x1f23a, 1}, + {0x1f240, 0x1f248, 1}, + {0x1f250, 0x1f251, 1}, + {0x1f300, 0x1f320, 1}, + {0x1f330, 0x1f335, 1}, + {0x1f337, 0x1f37c, 1}, + {0x1f380, 0x1f393, 1}, + {0x1f3a0, 0x1f3c4, 1}, + {0x1f3c6, 0x1f3ca, 1}, + {0x1f3e0, 0x1f3f0, 1}, + {0x1f400, 0x1f43e, 1}, + {0x1f440, 0x1f440, 1}, + {0x1f442, 0x1f4f7, 1}, + {0x1f4f9, 0x1f4fc, 1}, + {0x1f500, 0x1f53d, 1}, + {0x1f550, 0x1f567, 1}, + {0x1f5fb, 0x1f5ff, 1}, + {0x1f601, 0x1f610, 1}, + {0x1f612, 0x1f614, 1}, + {0x1f616, 0x1f616, 1}, + {0x1f618, 0x1f618, 1}, + {0x1f61a, 0x1f61a, 1}, + {0x1f61c, 0x1f61e, 1}, + {0x1f620, 0x1f625, 1}, + {0x1f628, 0x1f62b, 1}, + {0x1f62d, 0x1f62d, 1}, + {0x1f630, 0x1f633, 1}, + {0x1f635, 0x1f640, 1}, + {0x1f645, 0x1f64f, 1}, + {0x1f680, 0x1f6c5, 1}, + {0x1f700, 0x1f773, 1}, + {0xe0001, 0xe0001, 1}, + {0xe0020, 0xe007f, 1}, + }, +} + +var _Kannada = &RangeTable{ + R16: []Range16{ + {0x0c82, 0x0c83, 1}, + {0x0c85, 0x0c8c, 1}, + {0x0c8e, 0x0c90, 1}, + {0x0c92, 0x0ca8, 1}, + {0x0caa, 0x0cb3, 1}, + {0x0cb5, 0x0cb9, 1}, + {0x0cbc, 0x0cc4, 1}, + {0x0cc6, 0x0cc8, 1}, + {0x0cca, 0x0ccd, 1}, + {0x0cd5, 0x0cd6, 1}, + {0x0cde, 0x0cde, 1}, + {0x0ce0, 0x0ce3, 1}, + {0x0ce6, 0x0cef, 1}, + {0x0cf1, 0x0cf2, 1}, + }, +} + +var _Old_Turkic = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10c00, 0x10c48, 1}, + }, +} + +var _Tamil = &RangeTable{ + R16: []Range16{ + {0x0b82, 0x0b83, 1}, + {0x0b85, 0x0b8a, 1}, + {0x0b8e, 0x0b90, 1}, + {0x0b92, 0x0b95, 1}, + {0x0b99, 0x0b9a, 1}, + {0x0b9c, 0x0b9c, 1}, + {0x0b9e, 0x0b9f, 1}, + {0x0ba3, 0x0ba4, 1}, + {0x0ba8, 0x0baa, 1}, + {0x0bae, 0x0bb9, 1}, + {0x0bbe, 0x0bc2, 1}, + {0x0bc6, 0x0bc8, 1}, + {0x0bca, 0x0bcd, 1}, + {0x0bd0, 0x0bd0, 1}, + {0x0bd7, 0x0bd7, 1}, + {0x0be6, 0x0bfa, 1}, + }, +} + +var _Tagalog = &RangeTable{ + R16: []Range16{ + {0x1700, 0x170c, 1}, + {0x170e, 0x1714, 1}, + }, +} + +var _Brahmi = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x11000, 0x1104d, 1}, + {0x11052, 0x1106f, 1}, + }, +} + +var _Arabic = &RangeTable{ + R16: []Range16{ + {0x0600, 0x0603, 1}, + {0x0606, 0x060b, 1}, + {0x060d, 0x061a, 1}, + {0x061e, 0x061e, 1}, + {0x0620, 0x063f, 1}, + {0x0641, 0x064a, 1}, + {0x0656, 0x065e, 1}, + {0x066a, 0x066f, 1}, + {0x0671, 0x06dc, 1}, + {0x06de, 0x06ff, 1}, + {0x0750, 0x077f, 1}, + {0xfb50, 0xfbc1, 1}, + {0xfbd3, 0xfd3d, 1}, + {0xfd50, 0xfd8f, 1}, + {0xfd92, 0xfdc7, 1}, + {0xfdf0, 0xfdfc, 1}, + {0xfe70, 0xfe74, 1}, + {0xfe76, 0xfefc, 1}, + }, + R32: []Range32{ + {0x10e60, 0x10e7e, 1}, + }, +} + +var _Tagbanwa = &RangeTable{ + R16: []Range16{ + {0x1760, 0x176c, 1}, + {0x176e, 0x1770, 1}, + {0x1772, 0x1773, 1}, + }, +} + +var _Canadian_Aboriginal = &RangeTable{ + R16: []Range16{ + {0x1400, 0x167f, 1}, + {0x18b0, 0x18f5, 1}, + }, +} + +var _Tibetan = &RangeTable{ + R16: []Range16{ + {0x0f00, 0x0f47, 1}, + {0x0f49, 0x0f6c, 1}, + {0x0f71, 0x0f97, 1}, + {0x0f99, 0x0fbc, 1}, + {0x0fbe, 0x0fcc, 1}, + {0x0fce, 0x0fd4, 1}, + {0x0fd9, 0x0fda, 1}, + }, +} + +var _Coptic = &RangeTable{ + R16: []Range16{ + {0x03e2, 0x03ef, 1}, + {0x2c80, 0x2cf1, 1}, + {0x2cf9, 0x2cff, 1}, + }, +} + +var _Hiragana = &RangeTable{ + R16: []Range16{ + {0x3041, 0x3096, 1}, + {0x309d, 0x309f, 1}, + }, + R32: []Range32{ + {0x1b001, 0x1b001, 1}, + {0x1f200, 0x1f200, 1}, + }, +} + +var _Limbu = &RangeTable{ + R16: []Range16{ + {0x1900, 0x191c, 1}, + {0x1920, 0x192b, 1}, + {0x1930, 0x193b, 1}, + {0x1940, 0x1940, 1}, + {0x1944, 0x194f, 1}, + }, +} + +var _Egyptian_Hieroglyphs = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x13000, 0x1342e, 1}, + }, +} + +var _Avestan = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10b00, 0x10b35, 1}, + {0x10b39, 0x10b3f, 1}, + }, +} + +var _Myanmar = &RangeTable{ + R16: []Range16{ + {0x1000, 0x109f, 1}, + {0xaa60, 0xaa7b, 1}, + }, +} + +var _Armenian = &RangeTable{ + R16: []Range16{ + {0x0531, 0x0556, 1}, + {0x0559, 0x055f, 1}, + {0x0561, 0x0587, 1}, + {0x058a, 0x058a, 1}, + {0xfb13, 0xfb17, 1}, + }, +} + +var _Sinhala = &RangeTable{ + R16: []Range16{ + {0x0d82, 0x0d83, 1}, + {0x0d85, 0x0d96, 1}, + {0x0d9a, 0x0db1, 1}, + {0x0db3, 0x0dbb, 1}, + {0x0dbd, 0x0dbd, 1}, + {0x0dc0, 0x0dc6, 1}, + {0x0dca, 0x0dca, 1}, + {0x0dcf, 0x0dd4, 1}, + {0x0dd6, 0x0dd6, 1}, + {0x0dd8, 0x0ddf, 1}, + {0x0df2, 0x0df4, 1}, + }, +} + +var _Bengali = &RangeTable{ + R16: []Range16{ + {0x0981, 0x0983, 1}, + {0x0985, 0x098c, 1}, + {0x098f, 0x0990, 1}, + {0x0993, 0x09a8, 1}, + {0x09aa, 0x09b0, 1}, + {0x09b2, 0x09b2, 1}, + {0x09b6, 0x09b9, 1}, + {0x09bc, 0x09c4, 1}, + {0x09c7, 0x09c8, 1}, + {0x09cb, 0x09ce, 1}, + {0x09d7, 0x09d7, 1}, + {0x09dc, 0x09dd, 1}, + {0x09df, 0x09e3, 1}, + {0x09e6, 0x09fb, 1}, + }, +} + +var _Greek = &RangeTable{ + R16: []Range16{ + {0x0370, 0x0373, 1}, + {0x0375, 0x0377, 1}, + {0x037a, 0x037d, 1}, + {0x0384, 0x0384, 1}, + {0x0386, 0x0386, 1}, + {0x0388, 0x038a, 1}, + {0x038c, 0x038c, 1}, + {0x038e, 0x03a1, 1}, + {0x03a3, 0x03e1, 1}, + {0x03f0, 0x03ff, 1}, + {0x1d26, 0x1d2a, 1}, + {0x1d5d, 0x1d61, 1}, + {0x1d66, 0x1d6a, 1}, + {0x1dbf, 0x1dbf, 1}, + {0x1f00, 0x1f15, 1}, + {0x1f18, 0x1f1d, 1}, + {0x1f20, 0x1f45, 1}, + {0x1f48, 0x1f4d, 1}, + {0x1f50, 0x1f57, 1}, + {0x1f59, 0x1f59, 1}, + {0x1f5b, 0x1f5b, 1}, + {0x1f5d, 0x1f5d, 1}, + {0x1f5f, 0x1f7d, 1}, + {0x1f80, 0x1fb4, 1}, + {0x1fb6, 0x1fc4, 1}, + {0x1fc6, 0x1fd3, 1}, + {0x1fd6, 0x1fdb, 1}, + {0x1fdd, 0x1fef, 1}, + {0x1ff2, 0x1ff4, 1}, + {0x1ff6, 0x1ffe, 1}, + {0x2126, 0x2126, 1}, + }, + R32: []Range32{ + {0x10140, 0x1018a, 1}, + {0x1d200, 0x1d245, 1}, + }, +} + +var _Cham = &RangeTable{ + R16: []Range16{ + {0xaa00, 0xaa36, 1}, + {0xaa40, 0xaa4d, 1}, + {0xaa50, 0xaa59, 1}, + {0xaa5c, 0xaa5f, 1}, + }, +} + +var _Hebrew = &RangeTable{ + R16: []Range16{ + {0x0591, 0x05c7, 1}, + {0x05d0, 0x05ea, 1}, + {0x05f0, 0x05f4, 1}, + {0xfb1d, 0xfb36, 1}, + {0xfb38, 0xfb3c, 1}, + {0xfb3e, 0xfb3e, 1}, + {0xfb40, 0xfb41, 1}, + {0xfb43, 0xfb44, 1}, + {0xfb46, 0xfb4f, 1}, + }, +} + +var _Meetei_Mayek = &RangeTable{ + R16: []Range16{ + {0xabc0, 0xabed, 1}, + {0xabf0, 0xabf9, 1}, + }, +} + +var _Saurashtra = &RangeTable{ + R16: []Range16{ + {0xa880, 0xa8c4, 1}, + {0xa8ce, 0xa8d9, 1}, + }, +} + +var _Hangul = &RangeTable{ + R16: []Range16{ + {0x1100, 0x11ff, 1}, + {0x302e, 0x302f, 1}, + {0x3131, 0x318e, 1}, + {0x3200, 0x321e, 1}, + {0x3260, 0x327e, 1}, + {0xa960, 0xa97c, 1}, + {0xac00, 0xd7a3, 1}, + {0xd7b0, 0xd7c6, 1}, + {0xd7cb, 0xd7fb, 1}, + {0xffa0, 0xffbe, 1}, + {0xffc2, 0xffc7, 1}, + {0xffca, 0xffcf, 1}, + {0xffd2, 0xffd7, 1}, + {0xffda, 0xffdc, 1}, + }, +} + +var _Runic = &RangeTable{ + R16: []Range16{ + {0x16a0, 0x16ea, 1}, + {0x16ee, 0x16f0, 1}, + }, +} + +var _Deseret = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10400, 0x1044f, 1}, + }, +} + +var _Lisu = &RangeTable{ + R16: []Range16{ + {0xa4d0, 0xa4ff, 1}, + }, +} + +var _Sundanese = &RangeTable{ + R16: []Range16{ + {0x1b80, 0x1baa, 1}, + {0x1bae, 0x1bb9, 1}, + }, +} + +var _Glagolitic = &RangeTable{ + R16: []Range16{ + {0x2c00, 0x2c2e, 1}, + {0x2c30, 0x2c5e, 1}, + }, +} + +var _Oriya = &RangeTable{ + R16: []Range16{ + {0x0b01, 0x0b03, 1}, + {0x0b05, 0x0b0c, 1}, + {0x0b0f, 0x0b10, 1}, + {0x0b13, 0x0b28, 1}, + {0x0b2a, 0x0b30, 1}, + {0x0b32, 0x0b33, 1}, + {0x0b35, 0x0b39, 1}, + {0x0b3c, 0x0b44, 1}, + {0x0b47, 0x0b48, 1}, + {0x0b4b, 0x0b4d, 1}, + {0x0b56, 0x0b57, 1}, + {0x0b5c, 0x0b5d, 1}, + {0x0b5f, 0x0b63, 1}, + {0x0b66, 0x0b77, 1}, + }, +} + +var _Buhid = &RangeTable{ + R16: []Range16{ + {0x1740, 0x1753, 1}, + }, +} + +var _Ethiopic = &RangeTable{ + R16: []Range16{ + {0x1200, 0x1248, 1}, + {0x124a, 0x124d, 1}, + {0x1250, 0x1256, 1}, + {0x1258, 0x1258, 1}, + {0x125a, 0x125d, 1}, + {0x1260, 0x1288, 1}, + {0x128a, 0x128d, 1}, + {0x1290, 0x12b0, 1}, + {0x12b2, 0x12b5, 1}, + {0x12b8, 0x12be, 1}, + {0x12c0, 0x12c0, 1}, + {0x12c2, 0x12c5, 1}, + {0x12c8, 0x12d6, 1}, + {0x12d8, 0x1310, 1}, + {0x1312, 0x1315, 1}, + {0x1318, 0x135a, 1}, + {0x135d, 0x137c, 1}, + {0x1380, 0x1399, 1}, + {0x2d80, 0x2d96, 1}, + {0x2da0, 0x2da6, 1}, + {0x2da8, 0x2dae, 1}, + {0x2db0, 0x2db6, 1}, + {0x2db8, 0x2dbe, 1}, + {0x2dc0, 0x2dc6, 1}, + {0x2dc8, 0x2dce, 1}, + {0x2dd0, 0x2dd6, 1}, + {0x2dd8, 0x2dde, 1}, + {0xab01, 0xab06, 1}, + {0xab09, 0xab0e, 1}, + {0xab11, 0xab16, 1}, + {0xab20, 0xab26, 1}, + {0xab28, 0xab2e, 1}, + }, +} + +var _Javanese = &RangeTable{ + R16: []Range16{ + {0xa980, 0xa9cd, 1}, + {0xa9cf, 0xa9d9, 1}, + {0xa9de, 0xa9df, 1}, + }, +} + +var _Syloti_Nagri = &RangeTable{ + R16: []Range16{ + {0xa800, 0xa82b, 1}, + }, +} + +var _Vai = &RangeTable{ + R16: []Range16{ + {0xa500, 0xa62b, 1}, + }, +} + +var _Cherokee = &RangeTable{ + R16: []Range16{ + {0x13a0, 0x13f4, 1}, + }, +} + +var _Ogham = &RangeTable{ + R16: []Range16{ + {0x1680, 0x169c, 1}, + }, +} + +var _Batak = &RangeTable{ + R16: []Range16{ + {0x1bc0, 0x1bf3, 1}, + {0x1bfc, 0x1bff, 1}, + }, +} + +var _Syriac = &RangeTable{ + R16: []Range16{ + {0x0700, 0x070d, 1}, + {0x070f, 0x074a, 1}, + {0x074d, 0x074f, 1}, + }, +} + +var _Gurmukhi = &RangeTable{ + R16: []Range16{ + {0x0a01, 0x0a03, 1}, + {0x0a05, 0x0a0a, 1}, + {0x0a0f, 0x0a10, 1}, + {0x0a13, 0x0a28, 1}, + {0x0a2a, 0x0a30, 1}, + {0x0a32, 0x0a33, 1}, + {0x0a35, 0x0a36, 1}, + {0x0a38, 0x0a39, 1}, + {0x0a3c, 0x0a3c, 1}, + {0x0a3e, 0x0a42, 1}, + {0x0a47, 0x0a48, 1}, + {0x0a4b, 0x0a4d, 1}, + {0x0a51, 0x0a51, 1}, + {0x0a59, 0x0a5c, 1}, + {0x0a5e, 0x0a5e, 1}, + {0x0a66, 0x0a75, 1}, + }, +} + +var _Tai_Tham = &RangeTable{ + R16: []Range16{ + {0x1a20, 0x1a5e, 1}, + {0x1a60, 0x1a7c, 1}, + {0x1a7f, 0x1a89, 1}, + {0x1a90, 0x1a99, 1}, + {0x1aa0, 0x1aad, 1}, + }, +} + +var _Ol_Chiki = &RangeTable{ + R16: []Range16{ + {0x1c50, 0x1c7f, 1}, + }, +} + +var _Mongolian = &RangeTable{ + R16: []Range16{ + {0x1800, 0x1801, 1}, + {0x1804, 0x1804, 1}, + {0x1806, 0x180e, 1}, + {0x1810, 0x1819, 1}, + {0x1820, 0x1877, 1}, + {0x1880, 0x18aa, 1}, + }, +} + +var _Hanunoo = &RangeTable{ + R16: []Range16{ + {0x1720, 0x1734, 1}, + }, +} + +var _Cypriot = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10800, 0x10805, 1}, + {0x10808, 0x10808, 1}, + {0x1080a, 0x10835, 1}, + {0x10837, 0x10838, 1}, + {0x1083c, 0x1083c, 1}, + {0x1083f, 0x1083f, 1}, + }, +} + +var _Buginese = &RangeTable{ + R16: []Range16{ + {0x1a00, 0x1a1b, 1}, + {0x1a1e, 0x1a1f, 1}, + }, +} + +var _Bamum = &RangeTable{ + R16: []Range16{ + {0xa6a0, 0xa6f7, 1}, + }, + R32: []Range32{ + {0x16800, 0x16a38, 1}, + }, +} + +var _Lepcha = &RangeTable{ + R16: []Range16{ + {0x1c00, 0x1c37, 1}, + {0x1c3b, 0x1c49, 1}, + {0x1c4d, 0x1c4f, 1}, + }, +} + +var _Thaana = &RangeTable{ + R16: []Range16{ + {0x0780, 0x07b1, 1}, + }, +} + +var _Old_Persian = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x103a0, 0x103c3, 1}, + {0x103c8, 0x103d5, 1}, + }, +} + +var _Cuneiform = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x12000, 0x1236e, 1}, + {0x12400, 0x12462, 1}, + {0x12470, 0x12473, 1}, + }, +} + +var _Rejang = &RangeTable{ + R16: []Range16{ + {0xa930, 0xa953, 1}, + {0xa95f, 0xa95f, 1}, + }, +} + +var _Georgian = &RangeTable{ + R16: []Range16{ + {0x10a0, 0x10c5, 1}, + {0x10d0, 0x10fa, 1}, + {0x10fc, 0x10fc, 1}, + {0x2d00, 0x2d25, 1}, + }, +} + +var _Shavian = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10450, 0x1047f, 1}, + }, +} + +var _Lycian = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10280, 0x1029c, 1}, + }, +} + +var _Nko = &RangeTable{ + R16: []Range16{ + {0x07c0, 0x07fa, 1}, + }, +} + +var _Yi = &RangeTable{ + R16: []Range16{ + {0xa000, 0xa48c, 1}, + {0xa490, 0xa4c6, 1}, + }, +} + +var _Lao = &RangeTable{ + R16: []Range16{ + {0x0e81, 0x0e82, 1}, + {0x0e84, 0x0e84, 1}, + {0x0e87, 0x0e88, 1}, + {0x0e8a, 0x0e8a, 1}, + {0x0e8d, 0x0e8d, 1}, + {0x0e94, 0x0e97, 1}, + {0x0e99, 0x0e9f, 1}, + {0x0ea1, 0x0ea3, 1}, + {0x0ea5, 0x0ea5, 1}, + {0x0ea7, 0x0ea7, 1}, + {0x0eaa, 0x0eab, 1}, + {0x0ead, 0x0eb9, 1}, + {0x0ebb, 0x0ebd, 1}, + {0x0ec0, 0x0ec4, 1}, + {0x0ec6, 0x0ec6, 1}, + {0x0ec8, 0x0ecd, 1}, + {0x0ed0, 0x0ed9, 1}, + {0x0edc, 0x0edd, 1}, + }, +} + +var _Linear_B = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10000, 0x1000b, 1}, + {0x1000d, 0x10026, 1}, + {0x10028, 0x1003a, 1}, + {0x1003c, 0x1003d, 1}, + {0x1003f, 0x1004d, 1}, + {0x10050, 0x1005d, 1}, + {0x10080, 0x100fa, 1}, + }, +} + +var _Old_Italic = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10300, 0x1031e, 1}, + {0x10320, 0x10323, 1}, + }, +} + +var _Tai_Viet = &RangeTable{ + R16: []Range16{ + {0xaa80, 0xaac2, 1}, + {0xaadb, 0xaadf, 1}, + }, +} + +var _Devanagari = &RangeTable{ + R16: []Range16{ + {0x0900, 0x0950, 1}, + {0x0953, 0x0963, 1}, + {0x0966, 0x096f, 1}, + {0x0971, 0x0977, 1}, + {0x0979, 0x097f, 1}, + {0xa8e0, 0xa8fb, 1}, + }, +} + +var _Lydian = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10920, 0x10939, 1}, + {0x1093f, 0x1093f, 1}, + }, +} + +var _Tifinagh = &RangeTable{ + R16: []Range16{ + {0x2d30, 0x2d65, 1}, + {0x2d6f, 0x2d70, 1}, + {0x2d7f, 0x2d7f, 1}, + }, +} + +var _Ugaritic = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10380, 0x1039d, 1}, + {0x1039f, 0x1039f, 1}, + }, +} + +var _Thai = &RangeTable{ + R16: []Range16{ + {0x0e01, 0x0e3a, 1}, + {0x0e40, 0x0e5b, 1}, + }, +} + +var _Cyrillic = &RangeTable{ + R16: []Range16{ + {0x0400, 0x0484, 1}, + {0x0487, 0x0527, 1}, + {0x1d2b, 0x1d2b, 1}, + {0x1d78, 0x1d78, 1}, + {0x2de0, 0x2dff, 1}, + {0xa640, 0xa673, 1}, + {0xa67c, 0xa697, 1}, + }, +} + +var _Gujarati = &RangeTable{ + R16: []Range16{ + {0x0a81, 0x0a83, 1}, + {0x0a85, 0x0a8d, 1}, + {0x0a8f, 0x0a91, 1}, + {0x0a93, 0x0aa8, 1}, + {0x0aaa, 0x0ab0, 1}, + {0x0ab2, 0x0ab3, 1}, + {0x0ab5, 0x0ab9, 1}, + {0x0abc, 0x0ac5, 1}, + {0x0ac7, 0x0ac9, 1}, + {0x0acb, 0x0acd, 1}, + {0x0ad0, 0x0ad0, 1}, + {0x0ae0, 0x0ae3, 1}, + {0x0ae6, 0x0aef, 1}, + {0x0af1, 0x0af1, 1}, + }, +} + +var _Carian = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x102a0, 0x102d0, 1}, + }, +} + +var _Phoenician = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10900, 0x1091b, 1}, + {0x1091f, 0x1091f, 1}, + }, +} + +var _Balinese = &RangeTable{ + R16: []Range16{ + {0x1b00, 0x1b4b, 1}, + {0x1b50, 0x1b7c, 1}, + }, +} + +var _Braille = &RangeTable{ + R16: []Range16{ + {0x2800, 0x28ff, 1}, + }, +} + +var _Han = &RangeTable{ + R16: []Range16{ + {0x2e80, 0x2e99, 1}, + {0x2e9b, 0x2ef3, 1}, + {0x2f00, 0x2fd5, 1}, + {0x3005, 0x3005, 1}, + {0x3007, 0x3007, 1}, + {0x3021, 0x3029, 1}, + {0x3038, 0x303b, 1}, + {0x3400, 0x4db5, 1}, + {0x4e00, 0x9fcb, 1}, + {0xf900, 0xfa2d, 1}, + {0xfa30, 0xfa6d, 1}, + {0xfa70, 0xfad9, 1}, + }, + R32: []Range32{ + {0x20000, 0x2a6d6, 1}, + {0x2a700, 0x2b734, 1}, + {0x2b740, 0x2b81d, 1}, + {0x2f800, 0x2fa1d, 1}, + }, +} + +var _Gothic = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10330, 0x1034a, 1}, + }, } var ( @@ -3188,7 +4156,7 @@ var ( // DO NOT EDIT // Properties is the set of Unicode property tables. -var Properties = map[string][]Range{ +var Properties = map[string]*RangeTable{ "Pattern_Syntax": Pattern_Syntax, "Other_ID_Start": Other_ID_Start, "Pattern_White_Space": Pattern_White_Space, @@ -3223,837 +4191,927 @@ var Properties = map[string][]Range{ "White_Space": White_Space, } -var _Pattern_Syntax = []Range{ - {0x0021, 0x002f, 1}, - {0x003a, 0x0040, 1}, - {0x005b, 0x005e, 1}, - {0x0060, 0x0060, 1}, - {0x007b, 0x007e, 1}, - {0x00a1, 0x00a7, 1}, - {0x00a9, 0x00a9, 1}, - {0x00ab, 0x00ac, 1}, - {0x00ae, 0x00ae, 1}, - {0x00b0, 0x00b1, 1}, - {0x00b6, 0x00b6, 1}, - {0x00bb, 0x00bb, 1}, - {0x00bf, 0x00bf, 1}, - {0x00d7, 0x00d7, 1}, - {0x00f7, 0x00f7, 1}, - {0x2010, 0x2027, 1}, - {0x2030, 0x203e, 1}, - {0x2041, 0x2053, 1}, - {0x2055, 0x205e, 1}, - {0x2190, 0x245f, 1}, - {0x2500, 0x2775, 1}, - {0x2794, 0x2bff, 1}, - {0x2e00, 0x2e7f, 1}, - {0x3001, 0x3003, 1}, - {0x3008, 0x3020, 1}, - {0x3030, 0x3030, 1}, - {0xfd3e, 0xfd3f, 1}, - {0xfe45, 0xfe46, 1}, -} - -var _Other_ID_Start = []Range{ - {0x2118, 0x2118, 1}, - {0x212e, 0x212e, 1}, - {0x309b, 0x309c, 1}, -} - -var _Pattern_White_Space = []Range{ - {0x0009, 0x000d, 1}, - {0x0020, 0x0020, 1}, - {0x0085, 0x0085, 1}, - {0x200e, 0x200f, 1}, - {0x2028, 0x2029, 1}, -} - -var _Other_Lowercase = []Range{ - {0x02b0, 0x02b8, 1}, - {0x02c0, 0x02c1, 1}, - {0x02e0, 0x02e4, 1}, - {0x0345, 0x0345, 1}, - {0x037a, 0x037a, 1}, - {0x1d2c, 0x1d61, 1}, - {0x1d78, 0x1d78, 1}, - {0x1d9b, 0x1dbf, 1}, - {0x2090, 0x2094, 1}, - {0x2170, 0x217f, 1}, - {0x24d0, 0x24e9, 1}, - {0x2c7d, 0x2c7d, 1}, - {0xa770, 0xa770, 1}, -} - -var _Soft_Dotted = []Range{ - {0x0069, 0x006a, 1}, - {0x012f, 0x012f, 1}, - {0x0249, 0x0249, 1}, - {0x0268, 0x0268, 1}, - {0x029d, 0x029d, 1}, - {0x02b2, 0x02b2, 1}, - {0x03f3, 0x03f3, 1}, - {0x0456, 0x0456, 1}, - {0x0458, 0x0458, 1}, - {0x1d62, 0x1d62, 1}, - {0x1d96, 0x1d96, 1}, - {0x1da4, 0x1da4, 1}, - {0x1da8, 0x1da8, 1}, - {0x1e2d, 0x1e2d, 1}, - {0x1ecb, 0x1ecb, 1}, - {0x2071, 0x2071, 1}, - {0x2148, 0x2149, 1}, - {0x2c7c, 0x2c7c, 1}, - {0x1d422, 0x1d423, 1}, - {0x1d456, 0x1d457, 1}, - {0x1d48a, 0x1d48b, 1}, - {0x1d4be, 0x1d4bf, 1}, - {0x1d4f2, 0x1d4f3, 1}, - {0x1d526, 0x1d527, 1}, - {0x1d55a, 0x1d55b, 1}, - {0x1d58e, 0x1d58f, 1}, - {0x1d5c2, 0x1d5c3, 1}, - {0x1d5f6, 0x1d5f7, 1}, - {0x1d62a, 0x1d62b, 1}, - {0x1d65e, 0x1d65f, 1}, - {0x1d692, 0x1d693, 1}, -} - -var _Hex_Digit = []Range{ - {0x0030, 0x0039, 1}, - {0x0041, 0x0046, 1}, - {0x0061, 0x0066, 1}, - {0xff10, 0xff19, 1}, - {0xff21, 0xff26, 1}, - {0xff41, 0xff46, 1}, -} - -var _ASCII_Hex_Digit = []Range{ - {0x0030, 0x0039, 1}, - {0x0041, 0x0046, 1}, - {0x0061, 0x0066, 1}, -} - -var _Deprecated = []Range{ - {0x0149, 0x0149, 1}, - {0x0673, 0x0673, 1}, - {0x0f77, 0x0f77, 1}, - {0x0f79, 0x0f79, 1}, - {0x17a3, 0x17a4, 1}, - {0x206a, 0x206f, 1}, - {0x2329, 0x232a, 1}, - {0xe0001, 0xe0001, 1}, - {0xe0020, 0xe007f, 1}, -} - -var _Terminal_Punctuation = []Range{ - {0x0021, 0x0021, 1}, - {0x002c, 0x002c, 1}, - {0x002e, 0x002e, 1}, - {0x003a, 0x003b, 1}, - {0x003f, 0x003f, 1}, - {0x037e, 0x037e, 1}, - {0x0387, 0x0387, 1}, - {0x0589, 0x0589, 1}, - {0x05c3, 0x05c3, 1}, - {0x060c, 0x060c, 1}, - {0x061b, 0x061b, 1}, - {0x061f, 0x061f, 1}, - {0x06d4, 0x06d4, 1}, - {0x0700, 0x070a, 1}, - {0x070c, 0x070c, 1}, - {0x07f8, 0x07f9, 1}, - {0x0830, 0x083e, 1}, - {0x085e, 0x085e, 1}, - {0x0964, 0x0965, 1}, - {0x0e5a, 0x0e5b, 1}, - {0x0f08, 0x0f08, 1}, - {0x0f0d, 0x0f12, 1}, - {0x104a, 0x104b, 1}, - {0x1361, 0x1368, 1}, - {0x166d, 0x166e, 1}, - {0x16eb, 0x16ed, 1}, - {0x17d4, 0x17d6, 1}, - {0x17da, 0x17da, 1}, - {0x1802, 0x1805, 1}, - {0x1808, 0x1809, 1}, - {0x1944, 0x1945, 1}, - {0x1aa8, 0x1aab, 1}, - {0x1b5a, 0x1b5b, 1}, - {0x1b5d, 0x1b5f, 1}, - {0x1c3b, 0x1c3f, 1}, - {0x1c7e, 0x1c7f, 1}, - {0x203c, 0x203d, 1}, - {0x2047, 0x2049, 1}, - {0x2e2e, 0x2e2e, 1}, - {0x3001, 0x3002, 1}, - {0xa4fe, 0xa4ff, 1}, - {0xa60d, 0xa60f, 1}, - {0xa6f3, 0xa6f7, 1}, - {0xa876, 0xa877, 1}, - {0xa8ce, 0xa8cf, 1}, - {0xa92f, 0xa92f, 1}, - {0xa9c7, 0xa9c9, 1}, - {0xaa5d, 0xaa5f, 1}, - {0xaadf, 0xaadf, 1}, - {0xabeb, 0xabeb, 1}, - {0xfe50, 0xfe52, 1}, - {0xfe54, 0xfe57, 1}, - {0xff01, 0xff01, 1}, - {0xff0c, 0xff0c, 1}, - {0xff0e, 0xff0e, 1}, - {0xff1a, 0xff1b, 1}, - {0xff1f, 0xff1f, 1}, - {0xff61, 0xff61, 1}, - {0xff64, 0xff64, 1}, - {0x1039f, 0x1039f, 1}, - {0x103d0, 0x103d0, 1}, - {0x10857, 0x10857, 1}, - {0x1091f, 0x1091f, 1}, - {0x10b3a, 0x10b3f, 1}, - {0x11047, 0x1104d, 1}, - {0x110be, 0x110c1, 1}, - {0x12470, 0x12473, 1}, -} - -var _Quotation_Mark = []Range{ - {0x0022, 0x0022, 1}, - {0x0027, 0x0027, 1}, - {0x00ab, 0x00ab, 1}, - {0x00bb, 0x00bb, 1}, - {0x2018, 0x201f, 1}, - {0x2039, 0x203a, 1}, - {0x300c, 0x300f, 1}, - {0x301d, 0x301f, 1}, - {0xfe41, 0xfe44, 1}, - {0xff02, 0xff02, 1}, - {0xff07, 0xff07, 1}, - {0xff62, 0xff63, 1}, -} - -var _Other_ID_Continue = []Range{ - {0x00b7, 0x00b7, 1}, - {0x0387, 0x0387, 1}, - {0x1369, 0x1371, 1}, - {0x19da, 0x19da, 1}, -} - -var _Bidi_Control = []Range{ - {0x200e, 0x200f, 1}, - {0x202a, 0x202e, 1}, -} - -var _Variation_Selector = []Range{ - {0x180b, 0x180d, 1}, - {0xfe00, 0xfe0f, 1}, - {0xe0100, 0xe01ef, 1}, -} - -var _Noncharacter_Code_Point = []Range{ - {0xfdd0, 0xfdef, 1}, - {0xfffe, 0xffff, 1}, - {0x1fffe, 0x1ffff, 1}, - {0x2fffe, 0x2ffff, 1}, - {0x3fffe, 0x3ffff, 1}, - {0x4fffe, 0x4ffff, 1}, - {0x5fffe, 0x5ffff, 1}, - {0x6fffe, 0x6ffff, 1}, - {0x7fffe, 0x7ffff, 1}, - {0x8fffe, 0x8ffff, 1}, - {0x9fffe, 0x9ffff, 1}, - {0xafffe, 0xaffff, 1}, - {0xbfffe, 0xbffff, 1}, - {0xcfffe, 0xcffff, 1}, - {0xdfffe, 0xdffff, 1}, - {0xefffe, 0xeffff, 1}, - {0xffffe, 0xfffff, 1}, - {0x10fffe, 0x10ffff, 1}, -} - -var _Other_Math = []Range{ - {0x005e, 0x005e, 1}, - {0x03d0, 0x03d2, 1}, - {0x03d5, 0x03d5, 1}, - {0x03f0, 0x03f1, 1}, - {0x03f4, 0x03f5, 1}, - {0x2016, 0x2016, 1}, - {0x2032, 0x2034, 1}, - {0x2040, 0x2040, 1}, - {0x2061, 0x2064, 1}, - {0x207d, 0x207e, 1}, - {0x208d, 0x208e, 1}, - {0x20d0, 0x20dc, 1}, - {0x20e1, 0x20e1, 1}, - {0x20e5, 0x20e6, 1}, - {0x20eb, 0x20ef, 1}, - {0x2102, 0x2102, 1}, - {0x2107, 0x2107, 1}, - {0x210a, 0x2113, 1}, - {0x2115, 0x2115, 1}, - {0x2119, 0x211d, 1}, - {0x2124, 0x2124, 1}, - {0x2128, 0x2129, 1}, - {0x212c, 0x212d, 1}, - {0x212f, 0x2131, 1}, - {0x2133, 0x2138, 1}, - {0x213c, 0x213f, 1}, - {0x2145, 0x2149, 1}, - {0x2195, 0x2199, 1}, - {0x219c, 0x219f, 1}, - {0x21a1, 0x21a2, 1}, - {0x21a4, 0x21a5, 1}, - {0x21a7, 0x21a7, 1}, - {0x21a9, 0x21ad, 1}, - {0x21b0, 0x21b1, 1}, - {0x21b6, 0x21b7, 1}, - {0x21bc, 0x21cd, 1}, - {0x21d0, 0x21d1, 1}, - {0x21d3, 0x21d3, 1}, - {0x21d5, 0x21db, 1}, - {0x21dd, 0x21dd, 1}, - {0x21e4, 0x21e5, 1}, - {0x23b4, 0x23b5, 1}, - {0x23b7, 0x23b7, 1}, - {0x23d0, 0x23d0, 1}, - {0x23e2, 0x23e2, 1}, - {0x25a0, 0x25a1, 1}, - {0x25ae, 0x25b6, 1}, - {0x25bc, 0x25c0, 1}, - {0x25c6, 0x25c7, 1}, - {0x25ca, 0x25cb, 1}, - {0x25cf, 0x25d3, 1}, - {0x25e2, 0x25e2, 1}, - {0x25e4, 0x25e4, 1}, - {0x25e7, 0x25ec, 1}, - {0x2605, 0x2606, 1}, - {0x2640, 0x2640, 1}, - {0x2642, 0x2642, 1}, - {0x2660, 0x2663, 1}, - {0x266d, 0x266e, 1}, - {0x27c5, 0x27c6, 1}, - {0x27e6, 0x27ef, 1}, - {0x2983, 0x2998, 1}, - {0x29d8, 0x29db, 1}, - {0x29fc, 0x29fd, 1}, - {0xfe61, 0xfe61, 1}, - {0xfe63, 0xfe63, 1}, - {0xfe68, 0xfe68, 1}, - {0xff3c, 0xff3c, 1}, - {0xff3e, 0xff3e, 1}, - {0x1d400, 0x1d454, 1}, - {0x1d456, 0x1d49c, 1}, - {0x1d49e, 0x1d49f, 1}, - {0x1d4a2, 0x1d4a2, 1}, - {0x1d4a5, 0x1d4a6, 1}, - {0x1d4a9, 0x1d4ac, 1}, - {0x1d4ae, 0x1d4b9, 1}, - {0x1d4bb, 0x1d4bb, 1}, - {0x1d4bd, 0x1d4c3, 1}, - {0x1d4c5, 0x1d505, 1}, - {0x1d507, 0x1d50a, 1}, - {0x1d50d, 0x1d514, 1}, - {0x1d516, 0x1d51c, 1}, - {0x1d51e, 0x1d539, 1}, - {0x1d53b, 0x1d53e, 1}, - {0x1d540, 0x1d544, 1}, - {0x1d546, 0x1d546, 1}, - {0x1d54a, 0x1d550, 1}, - {0x1d552, 0x1d6a5, 1}, - {0x1d6a8, 0x1d6c0, 1}, - {0x1d6c2, 0x1d6da, 1}, - {0x1d6dc, 0x1d6fa, 1}, - {0x1d6fc, 0x1d714, 1}, - {0x1d716, 0x1d734, 1}, - {0x1d736, 0x1d74e, 1}, - {0x1d750, 0x1d76e, 1}, - {0x1d770, 0x1d788, 1}, - {0x1d78a, 0x1d7a8, 1}, - {0x1d7aa, 0x1d7c2, 1}, - {0x1d7c4, 0x1d7cb, 1}, - {0x1d7ce, 0x1d7ff, 1}, -} - -var _Unified_Ideograph = []Range{ - {0x3400, 0x4db5, 1}, - {0x4e00, 0x9fcb, 1}, - {0xfa0e, 0xfa0f, 1}, - {0xfa11, 0xfa11, 1}, - {0xfa13, 0xfa14, 1}, - {0xfa1f, 0xfa1f, 1}, - {0xfa21, 0xfa21, 1}, - {0xfa23, 0xfa24, 1}, - {0xfa27, 0xfa29, 1}, - {0x20000, 0x2a6d6, 1}, - {0x2a700, 0x2b734, 1}, - {0x2b740, 0x2b81d, 1}, -} - -var _Hyphen = []Range{ - {0x002d, 0x002d, 1}, - {0x00ad, 0x00ad, 1}, - {0x058a, 0x058a, 1}, - {0x1806, 0x1806, 1}, - {0x2010, 0x2011, 1}, - {0x2e17, 0x2e17, 1}, - {0x30fb, 0x30fb, 1}, - {0xfe63, 0xfe63, 1}, - {0xff0d, 0xff0d, 1}, - {0xff65, 0xff65, 1}, -} - -var _IDS_Binary_Operator = []Range{ - {0x2ff0, 0x2ff1, 1}, - {0x2ff4, 0x2ffb, 1}, -} - -var _Logical_Order_Exception = []Range{ - {0x0e40, 0x0e44, 1}, - {0x0ec0, 0x0ec4, 1}, - {0xaab5, 0xaab6, 1}, - {0xaab9, 0xaab9, 1}, - {0xaabb, 0xaabc, 1}, -} - -var _Radical = []Range{ - {0x2e80, 0x2e99, 1}, - {0x2e9b, 0x2ef3, 1}, - {0x2f00, 0x2fd5, 1}, -} - -var _Other_Uppercase = []Range{ - {0x2160, 0x216f, 1}, - {0x24b6, 0x24cf, 1}, -} - -var _STerm = []Range{ - {0x0021, 0x0021, 1}, - {0x002e, 0x002e, 1}, - {0x003f, 0x003f, 1}, - {0x055c, 0x055c, 1}, - {0x055e, 0x055e, 1}, - {0x0589, 0x0589, 1}, - {0x061f, 0x061f, 1}, - {0x06d4, 0x06d4, 1}, - {0x0700, 0x0702, 1}, - {0x07f9, 0x07f9, 1}, - {0x0964, 0x0965, 1}, - {0x104a, 0x104b, 1}, - {0x1362, 0x1362, 1}, - {0x1367, 0x1368, 1}, - {0x166e, 0x166e, 1}, - {0x1735, 0x1736, 1}, - {0x1803, 0x1803, 1}, - {0x1809, 0x1809, 1}, - {0x1944, 0x1945, 1}, - {0x1aa8, 0x1aab, 1}, - {0x1b5a, 0x1b5b, 1}, - {0x1b5e, 0x1b5f, 1}, - {0x1c3b, 0x1c3c, 1}, - {0x1c7e, 0x1c7f, 1}, - {0x203c, 0x203d, 1}, - {0x2047, 0x2049, 1}, - {0x2e2e, 0x2e2e, 1}, - {0x3002, 0x3002, 1}, - {0xa4ff, 0xa4ff, 1}, - {0xa60e, 0xa60f, 1}, - {0xa6f3, 0xa6f3, 1}, - {0xa6f7, 0xa6f7, 1}, - {0xa876, 0xa877, 1}, - {0xa8ce, 0xa8cf, 1}, - {0xa92f, 0xa92f, 1}, - {0xa9c8, 0xa9c9, 1}, - {0xaa5d, 0xaa5f, 1}, - {0xabeb, 0xabeb, 1}, - {0xfe52, 0xfe52, 1}, - {0xfe56, 0xfe57, 1}, - {0xff01, 0xff01, 1}, - {0xff0e, 0xff0e, 1}, - {0xff1f, 0xff1f, 1}, - {0xff61, 0xff61, 1}, - {0x10a56, 0x10a57, 1}, - {0x11047, 0x11048, 1}, - {0x110be, 0x110c1, 1}, -} - -var _Other_Alphabetic = []Range{ - {0x0345, 0x0345, 1}, - {0x05b0, 0x05bd, 1}, - {0x05bf, 0x05bf, 1}, - {0x05c1, 0x05c2, 1}, - {0x05c4, 0x05c5, 1}, - {0x05c7, 0x05c7, 1}, - {0x0610, 0x061a, 1}, - {0x064b, 0x0657, 1}, - {0x0659, 0x065f, 1}, - {0x0670, 0x0670, 1}, - {0x06d6, 0x06dc, 1}, - {0x06e1, 0x06e4, 1}, - {0x06e7, 0x06e8, 1}, - {0x06ed, 0x06ed, 1}, - {0x0711, 0x0711, 1}, - {0x0730, 0x073f, 1}, - {0x07a6, 0x07b0, 1}, - {0x0816, 0x0817, 1}, - {0x081b, 0x0823, 1}, - {0x0825, 0x0827, 1}, - {0x0829, 0x082c, 1}, - {0x0900, 0x0903, 1}, - {0x093a, 0x093b, 1}, - {0x093e, 0x094c, 1}, - {0x094e, 0x094f, 1}, - {0x0955, 0x0957, 1}, - {0x0962, 0x0963, 1}, - {0x0981, 0x0983, 1}, - {0x09be, 0x09c4, 1}, - {0x09c7, 0x09c8, 1}, - {0x09cb, 0x09cc, 1}, - {0x09d7, 0x09d7, 1}, - {0x09e2, 0x09e3, 1}, - {0x0a01, 0x0a03, 1}, - {0x0a3e, 0x0a42, 1}, - {0x0a47, 0x0a48, 1}, - {0x0a4b, 0x0a4c, 1}, - {0x0a51, 0x0a51, 1}, - {0x0a70, 0x0a71, 1}, - {0x0a75, 0x0a75, 1}, - {0x0a81, 0x0a83, 1}, - {0x0abe, 0x0ac5, 1}, - {0x0ac7, 0x0ac9, 1}, - {0x0acb, 0x0acc, 1}, - {0x0ae2, 0x0ae3, 1}, - {0x0b01, 0x0b03, 1}, - {0x0b3e, 0x0b44, 1}, - {0x0b47, 0x0b48, 1}, - {0x0b4b, 0x0b4c, 1}, - {0x0b56, 0x0b57, 1}, - {0x0b62, 0x0b63, 1}, - {0x0b82, 0x0b82, 1}, - {0x0bbe, 0x0bc2, 1}, - {0x0bc6, 0x0bc8, 1}, - {0x0bca, 0x0bcc, 1}, - {0x0bd7, 0x0bd7, 1}, - {0x0c01, 0x0c03, 1}, - {0x0c3e, 0x0c44, 1}, - {0x0c46, 0x0c48, 1}, - {0x0c4a, 0x0c4c, 1}, - {0x0c55, 0x0c56, 1}, - {0x0c62, 0x0c63, 1}, - {0x0c82, 0x0c83, 1}, - {0x0cbe, 0x0cc4, 1}, - {0x0cc6, 0x0cc8, 1}, - {0x0cca, 0x0ccc, 1}, - {0x0cd5, 0x0cd6, 1}, - {0x0ce2, 0x0ce3, 1}, - {0x0d02, 0x0d03, 1}, - {0x0d3e, 0x0d44, 1}, - {0x0d46, 0x0d48, 1}, - {0x0d4a, 0x0d4c, 1}, - {0x0d57, 0x0d57, 1}, - {0x0d62, 0x0d63, 1}, - {0x0d82, 0x0d83, 1}, - {0x0dcf, 0x0dd4, 1}, - {0x0dd6, 0x0dd6, 1}, - {0x0dd8, 0x0ddf, 1}, - {0x0df2, 0x0df3, 1}, - {0x0e31, 0x0e31, 1}, - {0x0e34, 0x0e3a, 1}, - {0x0e4d, 0x0e4d, 1}, - {0x0eb1, 0x0eb1, 1}, - {0x0eb4, 0x0eb9, 1}, - {0x0ebb, 0x0ebc, 1}, - {0x0ecd, 0x0ecd, 1}, - {0x0f71, 0x0f81, 1}, - {0x0f8d, 0x0f97, 1}, - {0x0f99, 0x0fbc, 1}, - {0x102b, 0x1036, 1}, - {0x1038, 0x1038, 1}, - {0x103b, 0x103e, 1}, - {0x1056, 0x1059, 1}, - {0x105e, 0x1060, 1}, - {0x1062, 0x1062, 1}, - {0x1067, 0x1068, 1}, - {0x1071, 0x1074, 1}, - {0x1082, 0x1086, 1}, - {0x109c, 0x109d, 1}, - {0x135f, 0x135f, 1}, - {0x1712, 0x1713, 1}, - {0x1732, 0x1733, 1}, - {0x1752, 0x1753, 1}, - {0x1772, 0x1773, 1}, - {0x17b6, 0x17c8, 1}, - {0x18a9, 0x18a9, 1}, - {0x1920, 0x192b, 1}, - {0x1930, 0x1938, 1}, - {0x19b0, 0x19c0, 1}, - {0x19c8, 0x19c9, 1}, - {0x1a17, 0x1a1b, 1}, - {0x1a55, 0x1a5e, 1}, - {0x1a61, 0x1a74, 1}, - {0x1b00, 0x1b04, 1}, - {0x1b35, 0x1b43, 1}, - {0x1b80, 0x1b82, 1}, - {0x1ba1, 0x1ba9, 1}, - {0x1be7, 0x1bf1, 1}, - {0x1c24, 0x1c35, 1}, - {0x1cf2, 0x1cf2, 1}, - {0x24b6, 0x24e9, 1}, - {0x2de0, 0x2dff, 1}, - {0xa823, 0xa827, 1}, - {0xa880, 0xa881, 1}, - {0xa8b4, 0xa8c3, 1}, - {0xa926, 0xa92a, 1}, - {0xa947, 0xa952, 1}, - {0xa980, 0xa983, 1}, - {0xa9b4, 0xa9bf, 1}, - {0xaa29, 0xaa36, 1}, - {0xaa43, 0xaa43, 1}, - {0xaa4c, 0xaa4d, 1}, - {0xaab0, 0xaab0, 1}, - {0xaab2, 0xaab4, 1}, - {0xaab7, 0xaab8, 1}, - {0xaabe, 0xaabe, 1}, - {0xabe3, 0xabea, 1}, - {0xfb1e, 0xfb1e, 1}, - {0x10a01, 0x10a03, 1}, - {0x10a05, 0x10a06, 1}, - {0x10a0c, 0x10a0f, 1}, - {0x11000, 0x11002, 1}, - {0x11038, 0x11045, 1}, - {0x11082, 0x11082, 1}, - {0x110b0, 0x110b8, 1}, -} - -var _Diacritic = []Range{ - {0x005e, 0x005e, 1}, - {0x0060, 0x0060, 1}, - {0x00a8, 0x00a8, 1}, - {0x00af, 0x00af, 1}, - {0x00b4, 0x00b4, 1}, - {0x00b7, 0x00b8, 1}, - {0x02b0, 0x034e, 1}, - {0x0350, 0x0357, 1}, - {0x035d, 0x0362, 1}, - {0x0374, 0x0375, 1}, - {0x037a, 0x037a, 1}, - {0x0384, 0x0385, 1}, - {0x0483, 0x0487, 1}, - {0x0559, 0x0559, 1}, - {0x0591, 0x05a1, 1}, - {0x05a3, 0x05bd, 1}, - {0x05bf, 0x05bf, 1}, - {0x05c1, 0x05c2, 1}, - {0x05c4, 0x05c4, 1}, - {0x064b, 0x0652, 1}, - {0x0657, 0x0658, 1}, - {0x06df, 0x06e0, 1}, - {0x06e5, 0x06e6, 1}, - {0x06ea, 0x06ec, 1}, - {0x0730, 0x074a, 1}, - {0x07a6, 0x07b0, 1}, - {0x07eb, 0x07f5, 1}, - {0x0818, 0x0819, 1}, - {0x093c, 0x093c, 1}, - {0x094d, 0x094d, 1}, - {0x0951, 0x0954, 1}, - {0x0971, 0x0971, 1}, - {0x09bc, 0x09bc, 1}, - {0x09cd, 0x09cd, 1}, - {0x0a3c, 0x0a3c, 1}, - {0x0a4d, 0x0a4d, 1}, - {0x0abc, 0x0abc, 1}, - {0x0acd, 0x0acd, 1}, - {0x0b3c, 0x0b3c, 1}, - {0x0b4d, 0x0b4d, 1}, - {0x0bcd, 0x0bcd, 1}, - {0x0c4d, 0x0c4d, 1}, - {0x0cbc, 0x0cbc, 1}, - {0x0ccd, 0x0ccd, 1}, - {0x0d4d, 0x0d4d, 1}, - {0x0dca, 0x0dca, 1}, - {0x0e47, 0x0e4c, 1}, - {0x0e4e, 0x0e4e, 1}, - {0x0ec8, 0x0ecc, 1}, - {0x0f18, 0x0f19, 1}, - {0x0f35, 0x0f35, 1}, - {0x0f37, 0x0f37, 1}, - {0x0f39, 0x0f39, 1}, - {0x0f3e, 0x0f3f, 1}, - {0x0f82, 0x0f84, 1}, - {0x0f86, 0x0f87, 1}, - {0x0fc6, 0x0fc6, 1}, - {0x1037, 0x1037, 1}, - {0x1039, 0x103a, 1}, - {0x1087, 0x108d, 1}, - {0x108f, 0x108f, 1}, - {0x109a, 0x109b, 1}, - {0x17c9, 0x17d3, 1}, - {0x17dd, 0x17dd, 1}, - {0x1939, 0x193b, 1}, - {0x1a75, 0x1a7c, 1}, - {0x1a7f, 0x1a7f, 1}, - {0x1b34, 0x1b34, 1}, - {0x1b44, 0x1b44, 1}, - {0x1b6b, 0x1b73, 1}, - {0x1baa, 0x1baa, 1}, - {0x1c36, 0x1c37, 1}, - {0x1c78, 0x1c7d, 1}, - {0x1cd0, 0x1ce8, 1}, - {0x1ced, 0x1ced, 1}, - {0x1d2c, 0x1d6a, 1}, - {0x1dc4, 0x1dcf, 1}, - {0x1dfd, 0x1dff, 1}, - {0x1fbd, 0x1fbd, 1}, - {0x1fbf, 0x1fc1, 1}, - {0x1fcd, 0x1fcf, 1}, - {0x1fdd, 0x1fdf, 1}, - {0x1fed, 0x1fef, 1}, - {0x1ffd, 0x1ffe, 1}, - {0x2cef, 0x2cf1, 1}, - {0x2e2f, 0x2e2f, 1}, - {0x302a, 0x302f, 1}, - {0x3099, 0x309c, 1}, - {0x30fc, 0x30fc, 1}, - {0xa66f, 0xa66f, 1}, - {0xa67c, 0xa67d, 1}, - {0xa67f, 0xa67f, 1}, - {0xa6f0, 0xa6f1, 1}, - {0xa717, 0xa721, 1}, - {0xa788, 0xa788, 1}, - {0xa8c4, 0xa8c4, 1}, - {0xa8e0, 0xa8f1, 1}, - {0xa92b, 0xa92e, 1}, - {0xa953, 0xa953, 1}, - {0xa9b3, 0xa9b3, 1}, - {0xa9c0, 0xa9c0, 1}, - {0xaa7b, 0xaa7b, 1}, - {0xaabf, 0xaac2, 1}, - {0xabec, 0xabed, 1}, - {0xfb1e, 0xfb1e, 1}, - {0xfe20, 0xfe26, 1}, - {0xff3e, 0xff3e, 1}, - {0xff40, 0xff40, 1}, - {0xff70, 0xff70, 1}, - {0xff9e, 0xff9f, 1}, - {0xffe3, 0xffe3, 1}, - {0x110b9, 0x110ba, 1}, - {0x1d167, 0x1d169, 1}, - {0x1d16d, 0x1d172, 1}, - {0x1d17b, 0x1d182, 1}, - {0x1d185, 0x1d18b, 1}, - {0x1d1aa, 0x1d1ad, 1}, -} - -var _Extender = []Range{ - {0x00b7, 0x00b7, 1}, - {0x02d0, 0x02d1, 1}, - {0x0640, 0x0640, 1}, - {0x07fa, 0x07fa, 1}, - {0x0e46, 0x0e46, 1}, - {0x0ec6, 0x0ec6, 1}, - {0x1843, 0x1843, 1}, - {0x1aa7, 0x1aa7, 1}, - {0x1c36, 0x1c36, 1}, - {0x1c7b, 0x1c7b, 1}, - {0x3005, 0x3005, 1}, - {0x3031, 0x3035, 1}, - {0x309d, 0x309e, 1}, - {0x30fc, 0x30fe, 1}, - {0xa015, 0xa015, 1}, - {0xa60c, 0xa60c, 1}, - {0xa9cf, 0xa9cf, 1}, - {0xaa70, 0xaa70, 1}, - {0xaadd, 0xaadd, 1}, - {0xff70, 0xff70, 1}, -} - -var _Join_Control = []Range{ - {0x200c, 0x200d, 1}, -} - -var _Ideographic = []Range{ - {0x3006, 0x3007, 1}, - {0x3021, 0x3029, 1}, - {0x3038, 0x303a, 1}, - {0x3400, 0x4db5, 1}, - {0x4e00, 0x9fcb, 1}, - {0xf900, 0xfa2d, 1}, - {0xfa30, 0xfa6d, 1}, - {0xfa70, 0xfad9, 1}, - {0x20000, 0x2a6d6, 1}, - {0x2a700, 0x2b734, 1}, - {0x2b740, 0x2b81d, 1}, - {0x2f800, 0x2fa1d, 1}, -} - -var _Dash = []Range{ - {0x002d, 0x002d, 1}, - {0x058a, 0x058a, 1}, - {0x05be, 0x05be, 1}, - {0x1400, 0x1400, 1}, - {0x1806, 0x1806, 1}, - {0x2010, 0x2015, 1}, - {0x2053, 0x2053, 1}, - {0x207b, 0x207b, 1}, - {0x208b, 0x208b, 1}, - {0x2212, 0x2212, 1}, - {0x2e17, 0x2e17, 1}, - {0x2e1a, 0x2e1a, 1}, - {0x301c, 0x301c, 1}, - {0x3030, 0x3030, 1}, - {0x30a0, 0x30a0, 1}, - {0xfe31, 0xfe32, 1}, - {0xfe58, 0xfe58, 1}, - {0xfe63, 0xfe63, 1}, - {0xff0d, 0xff0d, 1}, -} - -var _IDS_Trinary_Operator = []Range{ - {0x2ff2, 0x2ff3, 1}, -} - -var _Other_Grapheme_Extend = []Range{ - {0x09be, 0x09be, 1}, - {0x09d7, 0x09d7, 1}, - {0x0b3e, 0x0b3e, 1}, - {0x0b57, 0x0b57, 1}, - {0x0bbe, 0x0bbe, 1}, - {0x0bd7, 0x0bd7, 1}, - {0x0cc2, 0x0cc2, 1}, - {0x0cd5, 0x0cd6, 1}, - {0x0d3e, 0x0d3e, 1}, - {0x0d57, 0x0d57, 1}, - {0x0dcf, 0x0dcf, 1}, - {0x0ddf, 0x0ddf, 1}, - {0x200c, 0x200d, 1}, - {0xff9e, 0xff9f, 1}, - {0x1d165, 0x1d165, 1}, - {0x1d16e, 0x1d172, 1}, -} - -var _Other_Default_Ignorable_Code_Point = []Range{ - {0x034f, 0x034f, 1}, - {0x115f, 0x1160, 1}, - {0x2065, 0x2069, 1}, - {0x3164, 0x3164, 1}, - {0xffa0, 0xffa0, 1}, - {0xfff0, 0xfff8, 1}, - {0xe0000, 0xe0000, 1}, - {0xe0002, 0xe001f, 1}, - {0xe0080, 0xe00ff, 1}, - {0xe01f0, 0xe0fff, 1}, -} - -var _White_Space = []Range{ - {0x0009, 0x000d, 1}, - {0x0020, 0x0020, 1}, - {0x0085, 0x0085, 1}, - {0x00a0, 0x00a0, 1}, - {0x1680, 0x1680, 1}, - {0x180e, 0x180e, 1}, - {0x2000, 0x200a, 1}, - {0x2028, 0x2029, 1}, - {0x202f, 0x202f, 1}, - {0x205f, 0x205f, 1}, - {0x3000, 0x3000, 1}, +var _Pattern_Syntax = &RangeTable{ + R16: []Range16{ + {0x0021, 0x002f, 1}, + {0x003a, 0x0040, 1}, + {0x005b, 0x005e, 1}, + {0x0060, 0x0060, 1}, + {0x007b, 0x007e, 1}, + {0x00a1, 0x00a7, 1}, + {0x00a9, 0x00a9, 1}, + {0x00ab, 0x00ac, 1}, + {0x00ae, 0x00ae, 1}, + {0x00b0, 0x00b1, 1}, + {0x00b6, 0x00b6, 1}, + {0x00bb, 0x00bb, 1}, + {0x00bf, 0x00bf, 1}, + {0x00d7, 0x00d7, 1}, + {0x00f7, 0x00f7, 1}, + {0x2010, 0x2027, 1}, + {0x2030, 0x203e, 1}, + {0x2041, 0x2053, 1}, + {0x2055, 0x205e, 1}, + {0x2190, 0x245f, 1}, + {0x2500, 0x2775, 1}, + {0x2794, 0x2bff, 1}, + {0x2e00, 0x2e7f, 1}, + {0x3001, 0x3003, 1}, + {0x3008, 0x3020, 1}, + {0x3030, 0x3030, 1}, + {0xfd3e, 0xfd3f, 1}, + {0xfe45, 0xfe46, 1}, + }, +} + +var _Other_ID_Start = &RangeTable{ + R16: []Range16{ + {0x2118, 0x2118, 1}, + {0x212e, 0x212e, 1}, + {0x309b, 0x309c, 1}, + }, +} + +var _Pattern_White_Space = &RangeTable{ + R16: []Range16{ + {0x0009, 0x000d, 1}, + {0x0020, 0x0020, 1}, + {0x0085, 0x0085, 1}, + {0x200e, 0x200f, 1}, + {0x2028, 0x2029, 1}, + }, +} + +var _Other_Lowercase = &RangeTable{ + R16: []Range16{ + {0x02b0, 0x02b8, 1}, + {0x02c0, 0x02c1, 1}, + {0x02e0, 0x02e4, 1}, + {0x0345, 0x0345, 1}, + {0x037a, 0x037a, 1}, + {0x1d2c, 0x1d61, 1}, + {0x1d78, 0x1d78, 1}, + {0x1d9b, 0x1dbf, 1}, + {0x2090, 0x2094, 1}, + {0x2170, 0x217f, 1}, + {0x24d0, 0x24e9, 1}, + {0x2c7d, 0x2c7d, 1}, + {0xa770, 0xa770, 1}, + }, +} + +var _Soft_Dotted = &RangeTable{ + R16: []Range16{ + {0x0069, 0x006a, 1}, + {0x012f, 0x012f, 1}, + {0x0249, 0x0249, 1}, + {0x0268, 0x0268, 1}, + {0x029d, 0x029d, 1}, + {0x02b2, 0x02b2, 1}, + {0x03f3, 0x03f3, 1}, + {0x0456, 0x0456, 1}, + {0x0458, 0x0458, 1}, + {0x1d62, 0x1d62, 1}, + {0x1d96, 0x1d96, 1}, + {0x1da4, 0x1da4, 1}, + {0x1da8, 0x1da8, 1}, + {0x1e2d, 0x1e2d, 1}, + {0x1ecb, 0x1ecb, 1}, + {0x2071, 0x2071, 1}, + {0x2148, 0x2149, 1}, + {0x2c7c, 0x2c7c, 1}, + }, + R32: []Range32{ + {0x1d422, 0x1d423, 1}, + {0x1d456, 0x1d457, 1}, + {0x1d48a, 0x1d48b, 1}, + {0x1d4be, 0x1d4bf, 1}, + {0x1d4f2, 0x1d4f3, 1}, + {0x1d526, 0x1d527, 1}, + {0x1d55a, 0x1d55b, 1}, + {0x1d58e, 0x1d58f, 1}, + {0x1d5c2, 0x1d5c3, 1}, + {0x1d5f6, 0x1d5f7, 1}, + {0x1d62a, 0x1d62b, 1}, + {0x1d65e, 0x1d65f, 1}, + {0x1d692, 0x1d693, 1}, + }, +} + +var _Hex_Digit = &RangeTable{ + R16: []Range16{ + {0x0030, 0x0039, 1}, + {0x0041, 0x0046, 1}, + {0x0061, 0x0066, 1}, + {0xff10, 0xff19, 1}, + {0xff21, 0xff26, 1}, + {0xff41, 0xff46, 1}, + }, +} + +var _ASCII_Hex_Digit = &RangeTable{ + R16: []Range16{ + {0x0030, 0x0039, 1}, + {0x0041, 0x0046, 1}, + {0x0061, 0x0066, 1}, + }, +} + +var _Deprecated = &RangeTable{ + R16: []Range16{ + {0x0149, 0x0149, 1}, + {0x0673, 0x0673, 1}, + {0x0f77, 0x0f77, 1}, + {0x0f79, 0x0f79, 1}, + {0x17a3, 0x17a4, 1}, + {0x206a, 0x206f, 1}, + {0x2329, 0x232a, 1}, + }, + R32: []Range32{ + {0xe0001, 0xe0001, 1}, + {0xe0020, 0xe007f, 1}, + }, +} + +var _Terminal_Punctuation = &RangeTable{ + R16: []Range16{ + {0x0021, 0x0021, 1}, + {0x002c, 0x002c, 1}, + {0x002e, 0x002e, 1}, + {0x003a, 0x003b, 1}, + {0x003f, 0x003f, 1}, + {0x037e, 0x037e, 1}, + {0x0387, 0x0387, 1}, + {0x0589, 0x0589, 1}, + {0x05c3, 0x05c3, 1}, + {0x060c, 0x060c, 1}, + {0x061b, 0x061b, 1}, + {0x061f, 0x061f, 1}, + {0x06d4, 0x06d4, 1}, + {0x0700, 0x070a, 1}, + {0x070c, 0x070c, 1}, + {0x07f8, 0x07f9, 1}, + {0x0830, 0x083e, 1}, + {0x085e, 0x085e, 1}, + {0x0964, 0x0965, 1}, + {0x0e5a, 0x0e5b, 1}, + {0x0f08, 0x0f08, 1}, + {0x0f0d, 0x0f12, 1}, + {0x104a, 0x104b, 1}, + {0x1361, 0x1368, 1}, + {0x166d, 0x166e, 1}, + {0x16eb, 0x16ed, 1}, + {0x17d4, 0x17d6, 1}, + {0x17da, 0x17da, 1}, + {0x1802, 0x1805, 1}, + {0x1808, 0x1809, 1}, + {0x1944, 0x1945, 1}, + {0x1aa8, 0x1aab, 1}, + {0x1b5a, 0x1b5b, 1}, + {0x1b5d, 0x1b5f, 1}, + {0x1c3b, 0x1c3f, 1}, + {0x1c7e, 0x1c7f, 1}, + {0x203c, 0x203d, 1}, + {0x2047, 0x2049, 1}, + {0x2e2e, 0x2e2e, 1}, + {0x3001, 0x3002, 1}, + {0xa4fe, 0xa4ff, 1}, + {0xa60d, 0xa60f, 1}, + {0xa6f3, 0xa6f7, 1}, + {0xa876, 0xa877, 1}, + {0xa8ce, 0xa8cf, 1}, + {0xa92f, 0xa92f, 1}, + {0xa9c7, 0xa9c9, 1}, + {0xaa5d, 0xaa5f, 1}, + {0xaadf, 0xaadf, 1}, + {0xabeb, 0xabeb, 1}, + {0xfe50, 0xfe52, 1}, + {0xfe54, 0xfe57, 1}, + {0xff01, 0xff01, 1}, + {0xff0c, 0xff0c, 1}, + {0xff0e, 0xff0e, 1}, + {0xff1a, 0xff1b, 1}, + {0xff1f, 0xff1f, 1}, + {0xff61, 0xff61, 1}, + {0xff64, 0xff64, 1}, + }, + R32: []Range32{ + {0x1039f, 0x1039f, 1}, + {0x103d0, 0x103d0, 1}, + {0x10857, 0x10857, 1}, + {0x1091f, 0x1091f, 1}, + {0x10b3a, 0x10b3f, 1}, + {0x11047, 0x1104d, 1}, + {0x110be, 0x110c1, 1}, + {0x12470, 0x12473, 1}, + }, +} + +var _Quotation_Mark = &RangeTable{ + R16: []Range16{ + {0x0022, 0x0022, 1}, + {0x0027, 0x0027, 1}, + {0x00ab, 0x00ab, 1}, + {0x00bb, 0x00bb, 1}, + {0x2018, 0x201f, 1}, + {0x2039, 0x203a, 1}, + {0x300c, 0x300f, 1}, + {0x301d, 0x301f, 1}, + {0xfe41, 0xfe44, 1}, + {0xff02, 0xff02, 1}, + {0xff07, 0xff07, 1}, + {0xff62, 0xff63, 1}, + }, +} + +var _Other_ID_Continue = &RangeTable{ + R16: []Range16{ + {0x00b7, 0x00b7, 1}, + {0x0387, 0x0387, 1}, + {0x1369, 0x1371, 1}, + {0x19da, 0x19da, 1}, + }, +} + +var _Bidi_Control = &RangeTable{ + R16: []Range16{ + {0x200e, 0x200f, 1}, + {0x202a, 0x202e, 1}, + }, +} + +var _Variation_Selector = &RangeTable{ + R16: []Range16{ + {0x180b, 0x180d, 1}, + {0xfe00, 0xfe0f, 1}, + }, + R32: []Range32{ + {0xe0100, 0xe01ef, 1}, + }, +} + +var _Noncharacter_Code_Point = &RangeTable{ + R16: []Range16{ + {0xfdd0, 0xfdef, 1}, + {0xfffe, 0xffff, 1}, + }, + R32: []Range32{ + {0x1fffe, 0x1ffff, 1}, + {0x2fffe, 0x2ffff, 1}, + {0x3fffe, 0x3ffff, 1}, + {0x4fffe, 0x4ffff, 1}, + {0x5fffe, 0x5ffff, 1}, + {0x6fffe, 0x6ffff, 1}, + {0x7fffe, 0x7ffff, 1}, + {0x8fffe, 0x8ffff, 1}, + {0x9fffe, 0x9ffff, 1}, + {0xafffe, 0xaffff, 1}, + {0xbfffe, 0xbffff, 1}, + {0xcfffe, 0xcffff, 1}, + {0xdfffe, 0xdffff, 1}, + {0xefffe, 0xeffff, 1}, + {0xffffe, 0xfffff, 1}, + {0x10fffe, 0x10ffff, 1}, + }, +} + +var _Other_Math = &RangeTable{ + R16: []Range16{ + {0x005e, 0x005e, 1}, + {0x03d0, 0x03d2, 1}, + {0x03d5, 0x03d5, 1}, + {0x03f0, 0x03f1, 1}, + {0x03f4, 0x03f5, 1}, + {0x2016, 0x2016, 1}, + {0x2032, 0x2034, 1}, + {0x2040, 0x2040, 1}, + {0x2061, 0x2064, 1}, + {0x207d, 0x207e, 1}, + {0x208d, 0x208e, 1}, + {0x20d0, 0x20dc, 1}, + {0x20e1, 0x20e1, 1}, + {0x20e5, 0x20e6, 1}, + {0x20eb, 0x20ef, 1}, + {0x2102, 0x2102, 1}, + {0x2107, 0x2107, 1}, + {0x210a, 0x2113, 1}, + {0x2115, 0x2115, 1}, + {0x2119, 0x211d, 1}, + {0x2124, 0x2124, 1}, + {0x2128, 0x2129, 1}, + {0x212c, 0x212d, 1}, + {0x212f, 0x2131, 1}, + {0x2133, 0x2138, 1}, + {0x213c, 0x213f, 1}, + {0x2145, 0x2149, 1}, + {0x2195, 0x2199, 1}, + {0x219c, 0x219f, 1}, + {0x21a1, 0x21a2, 1}, + {0x21a4, 0x21a5, 1}, + {0x21a7, 0x21a7, 1}, + {0x21a9, 0x21ad, 1}, + {0x21b0, 0x21b1, 1}, + {0x21b6, 0x21b7, 1}, + {0x21bc, 0x21cd, 1}, + {0x21d0, 0x21d1, 1}, + {0x21d3, 0x21d3, 1}, + {0x21d5, 0x21db, 1}, + {0x21dd, 0x21dd, 1}, + {0x21e4, 0x21e5, 1}, + {0x23b4, 0x23b5, 1}, + {0x23b7, 0x23b7, 1}, + {0x23d0, 0x23d0, 1}, + {0x23e2, 0x23e2, 1}, + {0x25a0, 0x25a1, 1}, + {0x25ae, 0x25b6, 1}, + {0x25bc, 0x25c0, 1}, + {0x25c6, 0x25c7, 1}, + {0x25ca, 0x25cb, 1}, + {0x25cf, 0x25d3, 1}, + {0x25e2, 0x25e2, 1}, + {0x25e4, 0x25e4, 1}, + {0x25e7, 0x25ec, 1}, + {0x2605, 0x2606, 1}, + {0x2640, 0x2640, 1}, + {0x2642, 0x2642, 1}, + {0x2660, 0x2663, 1}, + {0x266d, 0x266e, 1}, + {0x27c5, 0x27c6, 1}, + {0x27e6, 0x27ef, 1}, + {0x2983, 0x2998, 1}, + {0x29d8, 0x29db, 1}, + {0x29fc, 0x29fd, 1}, + {0xfe61, 0xfe61, 1}, + {0xfe63, 0xfe63, 1}, + {0xfe68, 0xfe68, 1}, + {0xff3c, 0xff3c, 1}, + {0xff3e, 0xff3e, 1}, + }, + R32: []Range32{ + {0x1d400, 0x1d454, 1}, + {0x1d456, 0x1d49c, 1}, + {0x1d49e, 0x1d49f, 1}, + {0x1d4a2, 0x1d4a2, 1}, + {0x1d4a5, 0x1d4a6, 1}, + {0x1d4a9, 0x1d4ac, 1}, + {0x1d4ae, 0x1d4b9, 1}, + {0x1d4bb, 0x1d4bb, 1}, + {0x1d4bd, 0x1d4c3, 1}, + {0x1d4c5, 0x1d505, 1}, + {0x1d507, 0x1d50a, 1}, + {0x1d50d, 0x1d514, 1}, + {0x1d516, 0x1d51c, 1}, + {0x1d51e, 0x1d539, 1}, + {0x1d53b, 0x1d53e, 1}, + {0x1d540, 0x1d544, 1}, + {0x1d546, 0x1d546, 1}, + {0x1d54a, 0x1d550, 1}, + {0x1d552, 0x1d6a5, 1}, + {0x1d6a8, 0x1d6c0, 1}, + {0x1d6c2, 0x1d6da, 1}, + {0x1d6dc, 0x1d6fa, 1}, + {0x1d6fc, 0x1d714, 1}, + {0x1d716, 0x1d734, 1}, + {0x1d736, 0x1d74e, 1}, + {0x1d750, 0x1d76e, 1}, + {0x1d770, 0x1d788, 1}, + {0x1d78a, 0x1d7a8, 1}, + {0x1d7aa, 0x1d7c2, 1}, + {0x1d7c4, 0x1d7cb, 1}, + {0x1d7ce, 0x1d7ff, 1}, + }, +} + +var _Unified_Ideograph = &RangeTable{ + R16: []Range16{ + {0x3400, 0x4db5, 1}, + {0x4e00, 0x9fcb, 1}, + {0xfa0e, 0xfa0f, 1}, + {0xfa11, 0xfa11, 1}, + {0xfa13, 0xfa14, 1}, + {0xfa1f, 0xfa1f, 1}, + {0xfa21, 0xfa21, 1}, + {0xfa23, 0xfa24, 1}, + {0xfa27, 0xfa29, 1}, + }, + R32: []Range32{ + {0x20000, 0x2a6d6, 1}, + {0x2a700, 0x2b734, 1}, + {0x2b740, 0x2b81d, 1}, + }, +} + +var _Hyphen = &RangeTable{ + R16: []Range16{ + {0x002d, 0x002d, 1}, + {0x00ad, 0x00ad, 1}, + {0x058a, 0x058a, 1}, + {0x1806, 0x1806, 1}, + {0x2010, 0x2011, 1}, + {0x2e17, 0x2e17, 1}, + {0x30fb, 0x30fb, 1}, + {0xfe63, 0xfe63, 1}, + {0xff0d, 0xff0d, 1}, + {0xff65, 0xff65, 1}, + }, +} + +var _IDS_Binary_Operator = &RangeTable{ + R16: []Range16{ + {0x2ff0, 0x2ff1, 1}, + {0x2ff4, 0x2ffb, 1}, + }, +} + +var _Logical_Order_Exception = &RangeTable{ + R16: []Range16{ + {0x0e40, 0x0e44, 1}, + {0x0ec0, 0x0ec4, 1}, + {0xaab5, 0xaab6, 1}, + {0xaab9, 0xaab9, 1}, + {0xaabb, 0xaabc, 1}, + }, +} + +var _Radical = &RangeTable{ + R16: []Range16{ + {0x2e80, 0x2e99, 1}, + {0x2e9b, 0x2ef3, 1}, + {0x2f00, 0x2fd5, 1}, + }, +} + +var _Other_Uppercase = &RangeTable{ + R16: []Range16{ + {0x2160, 0x216f, 1}, + {0x24b6, 0x24cf, 1}, + }, +} + +var _STerm = &RangeTable{ + R16: []Range16{ + {0x0021, 0x0021, 1}, + {0x002e, 0x002e, 1}, + {0x003f, 0x003f, 1}, + {0x055c, 0x055c, 1}, + {0x055e, 0x055e, 1}, + {0x0589, 0x0589, 1}, + {0x061f, 0x061f, 1}, + {0x06d4, 0x06d4, 1}, + {0x0700, 0x0702, 1}, + {0x07f9, 0x07f9, 1}, + {0x0964, 0x0965, 1}, + {0x104a, 0x104b, 1}, + {0x1362, 0x1362, 1}, + {0x1367, 0x1368, 1}, + {0x166e, 0x166e, 1}, + {0x1735, 0x1736, 1}, + {0x1803, 0x1803, 1}, + {0x1809, 0x1809, 1}, + {0x1944, 0x1945, 1}, + {0x1aa8, 0x1aab, 1}, + {0x1b5a, 0x1b5b, 1}, + {0x1b5e, 0x1b5f, 1}, + {0x1c3b, 0x1c3c, 1}, + {0x1c7e, 0x1c7f, 1}, + {0x203c, 0x203d, 1}, + {0x2047, 0x2049, 1}, + {0x2e2e, 0x2e2e, 1}, + {0x3002, 0x3002, 1}, + {0xa4ff, 0xa4ff, 1}, + {0xa60e, 0xa60f, 1}, + {0xa6f3, 0xa6f3, 1}, + {0xa6f7, 0xa6f7, 1}, + {0xa876, 0xa877, 1}, + {0xa8ce, 0xa8cf, 1}, + {0xa92f, 0xa92f, 1}, + {0xa9c8, 0xa9c9, 1}, + {0xaa5d, 0xaa5f, 1}, + {0xabeb, 0xabeb, 1}, + {0xfe52, 0xfe52, 1}, + {0xfe56, 0xfe57, 1}, + {0xff01, 0xff01, 1}, + {0xff0e, 0xff0e, 1}, + {0xff1f, 0xff1f, 1}, + {0xff61, 0xff61, 1}, + }, + R32: []Range32{ + {0x10a56, 0x10a57, 1}, + {0x11047, 0x11048, 1}, + {0x110be, 0x110c1, 1}, + }, +} + +var _Other_Alphabetic = &RangeTable{ + R16: []Range16{ + {0x0345, 0x0345, 1}, + {0x05b0, 0x05bd, 1}, + {0x05bf, 0x05bf, 1}, + {0x05c1, 0x05c2, 1}, + {0x05c4, 0x05c5, 1}, + {0x05c7, 0x05c7, 1}, + {0x0610, 0x061a, 1}, + {0x064b, 0x0657, 1}, + {0x0659, 0x065f, 1}, + {0x0670, 0x0670, 1}, + {0x06d6, 0x06dc, 1}, + {0x06e1, 0x06e4, 1}, + {0x06e7, 0x06e8, 1}, + {0x06ed, 0x06ed, 1}, + {0x0711, 0x0711, 1}, + {0x0730, 0x073f, 1}, + {0x07a6, 0x07b0, 1}, + {0x0816, 0x0817, 1}, + {0x081b, 0x0823, 1}, + {0x0825, 0x0827, 1}, + {0x0829, 0x082c, 1}, + {0x0900, 0x0903, 1}, + {0x093a, 0x093b, 1}, + {0x093e, 0x094c, 1}, + {0x094e, 0x094f, 1}, + {0x0955, 0x0957, 1}, + {0x0962, 0x0963, 1}, + {0x0981, 0x0983, 1}, + {0x09be, 0x09c4, 1}, + {0x09c7, 0x09c8, 1}, + {0x09cb, 0x09cc, 1}, + {0x09d7, 0x09d7, 1}, + {0x09e2, 0x09e3, 1}, + {0x0a01, 0x0a03, 1}, + {0x0a3e, 0x0a42, 1}, + {0x0a47, 0x0a48, 1}, + {0x0a4b, 0x0a4c, 1}, + {0x0a51, 0x0a51, 1}, + {0x0a70, 0x0a71, 1}, + {0x0a75, 0x0a75, 1}, + {0x0a81, 0x0a83, 1}, + {0x0abe, 0x0ac5, 1}, + {0x0ac7, 0x0ac9, 1}, + {0x0acb, 0x0acc, 1}, + {0x0ae2, 0x0ae3, 1}, + {0x0b01, 0x0b03, 1}, + {0x0b3e, 0x0b44, 1}, + {0x0b47, 0x0b48, 1}, + {0x0b4b, 0x0b4c, 1}, + {0x0b56, 0x0b57, 1}, + {0x0b62, 0x0b63, 1}, + {0x0b82, 0x0b82, 1}, + {0x0bbe, 0x0bc2, 1}, + {0x0bc6, 0x0bc8, 1}, + {0x0bca, 0x0bcc, 1}, + {0x0bd7, 0x0bd7, 1}, + {0x0c01, 0x0c03, 1}, + {0x0c3e, 0x0c44, 1}, + {0x0c46, 0x0c48, 1}, + {0x0c4a, 0x0c4c, 1}, + {0x0c55, 0x0c56, 1}, + {0x0c62, 0x0c63, 1}, + {0x0c82, 0x0c83, 1}, + {0x0cbe, 0x0cc4, 1}, + {0x0cc6, 0x0cc8, 1}, + {0x0cca, 0x0ccc, 1}, + {0x0cd5, 0x0cd6, 1}, + {0x0ce2, 0x0ce3, 1}, + {0x0d02, 0x0d03, 1}, + {0x0d3e, 0x0d44, 1}, + {0x0d46, 0x0d48, 1}, + {0x0d4a, 0x0d4c, 1}, + {0x0d57, 0x0d57, 1}, + {0x0d62, 0x0d63, 1}, + {0x0d82, 0x0d83, 1}, + {0x0dcf, 0x0dd4, 1}, + {0x0dd6, 0x0dd6, 1}, + {0x0dd8, 0x0ddf, 1}, + {0x0df2, 0x0df3, 1}, + {0x0e31, 0x0e31, 1}, + {0x0e34, 0x0e3a, 1}, + {0x0e4d, 0x0e4d, 1}, + {0x0eb1, 0x0eb1, 1}, + {0x0eb4, 0x0eb9, 1}, + {0x0ebb, 0x0ebc, 1}, + {0x0ecd, 0x0ecd, 1}, + {0x0f71, 0x0f81, 1}, + {0x0f8d, 0x0f97, 1}, + {0x0f99, 0x0fbc, 1}, + {0x102b, 0x1036, 1}, + {0x1038, 0x1038, 1}, + {0x103b, 0x103e, 1}, + {0x1056, 0x1059, 1}, + {0x105e, 0x1060, 1}, + {0x1062, 0x1062, 1}, + {0x1067, 0x1068, 1}, + {0x1071, 0x1074, 1}, + {0x1082, 0x1086, 1}, + {0x109c, 0x109d, 1}, + {0x135f, 0x135f, 1}, + {0x1712, 0x1713, 1}, + {0x1732, 0x1733, 1}, + {0x1752, 0x1753, 1}, + {0x1772, 0x1773, 1}, + {0x17b6, 0x17c8, 1}, + {0x18a9, 0x18a9, 1}, + {0x1920, 0x192b, 1}, + {0x1930, 0x1938, 1}, + {0x19b0, 0x19c0, 1}, + {0x19c8, 0x19c9, 1}, + {0x1a17, 0x1a1b, 1}, + {0x1a55, 0x1a5e, 1}, + {0x1a61, 0x1a74, 1}, + {0x1b00, 0x1b04, 1}, + {0x1b35, 0x1b43, 1}, + {0x1b80, 0x1b82, 1}, + {0x1ba1, 0x1ba9, 1}, + {0x1be7, 0x1bf1, 1}, + {0x1c24, 0x1c35, 1}, + {0x1cf2, 0x1cf2, 1}, + {0x24b6, 0x24e9, 1}, + {0x2de0, 0x2dff, 1}, + {0xa823, 0xa827, 1}, + {0xa880, 0xa881, 1}, + {0xa8b4, 0xa8c3, 1}, + {0xa926, 0xa92a, 1}, + {0xa947, 0xa952, 1}, + {0xa980, 0xa983, 1}, + {0xa9b4, 0xa9bf, 1}, + {0xaa29, 0xaa36, 1}, + {0xaa43, 0xaa43, 1}, + {0xaa4c, 0xaa4d, 1}, + {0xaab0, 0xaab0, 1}, + {0xaab2, 0xaab4, 1}, + {0xaab7, 0xaab8, 1}, + {0xaabe, 0xaabe, 1}, + {0xabe3, 0xabea, 1}, + {0xfb1e, 0xfb1e, 1}, + }, + R32: []Range32{ + {0x10a01, 0x10a03, 1}, + {0x10a05, 0x10a06, 1}, + {0x10a0c, 0x10a0f, 1}, + {0x11000, 0x11002, 1}, + {0x11038, 0x11045, 1}, + {0x11082, 0x11082, 1}, + {0x110b0, 0x110b8, 1}, + }, +} + +var _Diacritic = &RangeTable{ + R16: []Range16{ + {0x005e, 0x005e, 1}, + {0x0060, 0x0060, 1}, + {0x00a8, 0x00a8, 1}, + {0x00af, 0x00af, 1}, + {0x00b4, 0x00b4, 1}, + {0x00b7, 0x00b8, 1}, + {0x02b0, 0x034e, 1}, + {0x0350, 0x0357, 1}, + {0x035d, 0x0362, 1}, + {0x0374, 0x0375, 1}, + {0x037a, 0x037a, 1}, + {0x0384, 0x0385, 1}, + {0x0483, 0x0487, 1}, + {0x0559, 0x0559, 1}, + {0x0591, 0x05a1, 1}, + {0x05a3, 0x05bd, 1}, + {0x05bf, 0x05bf, 1}, + {0x05c1, 0x05c2, 1}, + {0x05c4, 0x05c4, 1}, + {0x064b, 0x0652, 1}, + {0x0657, 0x0658, 1}, + {0x06df, 0x06e0, 1}, + {0x06e5, 0x06e6, 1}, + {0x06ea, 0x06ec, 1}, + {0x0730, 0x074a, 1}, + {0x07a6, 0x07b0, 1}, + {0x07eb, 0x07f5, 1}, + {0x0818, 0x0819, 1}, + {0x093c, 0x093c, 1}, + {0x094d, 0x094d, 1}, + {0x0951, 0x0954, 1}, + {0x0971, 0x0971, 1}, + {0x09bc, 0x09bc, 1}, + {0x09cd, 0x09cd, 1}, + {0x0a3c, 0x0a3c, 1}, + {0x0a4d, 0x0a4d, 1}, + {0x0abc, 0x0abc, 1}, + {0x0acd, 0x0acd, 1}, + {0x0b3c, 0x0b3c, 1}, + {0x0b4d, 0x0b4d, 1}, + {0x0bcd, 0x0bcd, 1}, + {0x0c4d, 0x0c4d, 1}, + {0x0cbc, 0x0cbc, 1}, + {0x0ccd, 0x0ccd, 1}, + {0x0d4d, 0x0d4d, 1}, + {0x0dca, 0x0dca, 1}, + {0x0e47, 0x0e4c, 1}, + {0x0e4e, 0x0e4e, 1}, + {0x0ec8, 0x0ecc, 1}, + {0x0f18, 0x0f19, 1}, + {0x0f35, 0x0f35, 1}, + {0x0f37, 0x0f37, 1}, + {0x0f39, 0x0f39, 1}, + {0x0f3e, 0x0f3f, 1}, + {0x0f82, 0x0f84, 1}, + {0x0f86, 0x0f87, 1}, + {0x0fc6, 0x0fc6, 1}, + {0x1037, 0x1037, 1}, + {0x1039, 0x103a, 1}, + {0x1087, 0x108d, 1}, + {0x108f, 0x108f, 1}, + {0x109a, 0x109b, 1}, + {0x17c9, 0x17d3, 1}, + {0x17dd, 0x17dd, 1}, + {0x1939, 0x193b, 1}, + {0x1a75, 0x1a7c, 1}, + {0x1a7f, 0x1a7f, 1}, + {0x1b34, 0x1b34, 1}, + {0x1b44, 0x1b44, 1}, + {0x1b6b, 0x1b73, 1}, + {0x1baa, 0x1baa, 1}, + {0x1c36, 0x1c37, 1}, + {0x1c78, 0x1c7d, 1}, + {0x1cd0, 0x1ce8, 1}, + {0x1ced, 0x1ced, 1}, + {0x1d2c, 0x1d6a, 1}, + {0x1dc4, 0x1dcf, 1}, + {0x1dfd, 0x1dff, 1}, + {0x1fbd, 0x1fbd, 1}, + {0x1fbf, 0x1fc1, 1}, + {0x1fcd, 0x1fcf, 1}, + {0x1fdd, 0x1fdf, 1}, + {0x1fed, 0x1fef, 1}, + {0x1ffd, 0x1ffe, 1}, + {0x2cef, 0x2cf1, 1}, + {0x2e2f, 0x2e2f, 1}, + {0x302a, 0x302f, 1}, + {0x3099, 0x309c, 1}, + {0x30fc, 0x30fc, 1}, + {0xa66f, 0xa66f, 1}, + {0xa67c, 0xa67d, 1}, + {0xa67f, 0xa67f, 1}, + {0xa6f0, 0xa6f1, 1}, + {0xa717, 0xa721, 1}, + {0xa788, 0xa788, 1}, + {0xa8c4, 0xa8c4, 1}, + {0xa8e0, 0xa8f1, 1}, + {0xa92b, 0xa92e, 1}, + {0xa953, 0xa953, 1}, + {0xa9b3, 0xa9b3, 1}, + {0xa9c0, 0xa9c0, 1}, + {0xaa7b, 0xaa7b, 1}, + {0xaabf, 0xaac2, 1}, + {0xabec, 0xabed, 1}, + {0xfb1e, 0xfb1e, 1}, + {0xfe20, 0xfe26, 1}, + {0xff3e, 0xff3e, 1}, + {0xff40, 0xff40, 1}, + {0xff70, 0xff70, 1}, + {0xff9e, 0xff9f, 1}, + {0xffe3, 0xffe3, 1}, + }, + R32: []Range32{ + {0x110b9, 0x110ba, 1}, + {0x1d167, 0x1d169, 1}, + {0x1d16d, 0x1d172, 1}, + {0x1d17b, 0x1d182, 1}, + {0x1d185, 0x1d18b, 1}, + {0x1d1aa, 0x1d1ad, 1}, + }, +} + +var _Extender = &RangeTable{ + R16: []Range16{ + {0x00b7, 0x00b7, 1}, + {0x02d0, 0x02d1, 1}, + {0x0640, 0x0640, 1}, + {0x07fa, 0x07fa, 1}, + {0x0e46, 0x0e46, 1}, + {0x0ec6, 0x0ec6, 1}, + {0x1843, 0x1843, 1}, + {0x1aa7, 0x1aa7, 1}, + {0x1c36, 0x1c36, 1}, + {0x1c7b, 0x1c7b, 1}, + {0x3005, 0x3005, 1}, + {0x3031, 0x3035, 1}, + {0x309d, 0x309e, 1}, + {0x30fc, 0x30fe, 1}, + {0xa015, 0xa015, 1}, + {0xa60c, 0xa60c, 1}, + {0xa9cf, 0xa9cf, 1}, + {0xaa70, 0xaa70, 1}, + {0xaadd, 0xaadd, 1}, + {0xff70, 0xff70, 1}, + }, +} + +var _Join_Control = &RangeTable{ + R16: []Range16{ + {0x200c, 0x200d, 1}, + }, +} + +var _Ideographic = &RangeTable{ + R16: []Range16{ + {0x3006, 0x3007, 1}, + {0x3021, 0x3029, 1}, + {0x3038, 0x303a, 1}, + {0x3400, 0x4db5, 1}, + {0x4e00, 0x9fcb, 1}, + {0xf900, 0xfa2d, 1}, + {0xfa30, 0xfa6d, 1}, + {0xfa70, 0xfad9, 1}, + }, + R32: []Range32{ + {0x20000, 0x2a6d6, 1}, + {0x2a700, 0x2b734, 1}, + {0x2b740, 0x2b81d, 1}, + {0x2f800, 0x2fa1d, 1}, + }, +} + +var _Dash = &RangeTable{ + R16: []Range16{ + {0x002d, 0x002d, 1}, + {0x058a, 0x058a, 1}, + {0x05be, 0x05be, 1}, + {0x1400, 0x1400, 1}, + {0x1806, 0x1806, 1}, + {0x2010, 0x2015, 1}, + {0x2053, 0x2053, 1}, + {0x207b, 0x207b, 1}, + {0x208b, 0x208b, 1}, + {0x2212, 0x2212, 1}, + {0x2e17, 0x2e17, 1}, + {0x2e1a, 0x2e1a, 1}, + {0x301c, 0x301c, 1}, + {0x3030, 0x3030, 1}, + {0x30a0, 0x30a0, 1}, + {0xfe31, 0xfe32, 1}, + {0xfe58, 0xfe58, 1}, + {0xfe63, 0xfe63, 1}, + {0xff0d, 0xff0d, 1}, + }, +} + +var _IDS_Trinary_Operator = &RangeTable{ + R16: []Range16{ + {0x2ff2, 0x2ff3, 1}, + }, +} + +var _Other_Grapheme_Extend = &RangeTable{ + R16: []Range16{ + {0x09be, 0x09be, 1}, + {0x09d7, 0x09d7, 1}, + {0x0b3e, 0x0b3e, 1}, + {0x0b57, 0x0b57, 1}, + {0x0bbe, 0x0bbe, 1}, + {0x0bd7, 0x0bd7, 1}, + {0x0cc2, 0x0cc2, 1}, + {0x0cd5, 0x0cd6, 1}, + {0x0d3e, 0x0d3e, 1}, + {0x0d57, 0x0d57, 1}, + {0x0dcf, 0x0dcf, 1}, + {0x0ddf, 0x0ddf, 1}, + {0x200c, 0x200d, 1}, + {0xff9e, 0xff9f, 1}, + }, + R32: []Range32{ + {0x1d165, 0x1d165, 1}, + {0x1d16e, 0x1d172, 1}, + }, +} + +var _Other_Default_Ignorable_Code_Point = &RangeTable{ + R16: []Range16{ + {0x034f, 0x034f, 1}, + {0x115f, 0x1160, 1}, + {0x2065, 0x2069, 1}, + {0x3164, 0x3164, 1}, + {0xffa0, 0xffa0, 1}, + {0xfff0, 0xfff8, 1}, + }, + R32: []Range32{ + {0xe0000, 0xe0000, 1}, + {0xe0002, 0xe001f, 1}, + {0xe0080, 0xe00ff, 1}, + {0xe01f0, 0xe0fff, 1}, + }, +} + +var _White_Space = &RangeTable{ + R16: []Range16{ + {0x0009, 0x000d, 1}, + {0x0020, 0x0020, 1}, + {0x0085, 0x0085, 1}, + {0x00a0, 0x00a0, 1}, + {0x1680, 0x1680, 1}, + {0x180e, 0x180e, 1}, + {0x2000, 0x200a, 1}, + {0x2028, 0x2029, 1}, + {0x202f, 0x202f, 1}, + {0x205f, 0x205f, 1}, + {0x3000, 0x3000, 1}, + }, } var ( @@ -4353,3 +5411,264 @@ var _CaseRanges = []CaseRange{ {0x10400, 0x10427, d{0, 40, 0}}, {0x10428, 0x1044F, d{-40, 0, -40}}, } +var properties = [MaxLatin1 + 1]uint8{ + 0x00: pC, // '\x00' + 0x01: pC, // '\x01' + 0x02: pC, // '\x02' + 0x03: pC, // '\x03' + 0x04: pC, // '\x04' + 0x05: pC, // '\x05' + 0x06: pC, // '\x06' + 0x07: pC, // '\a' + 0x08: pC, // '\b' + 0x09: pC, // '\t' + 0x0A: pC, // '\n' + 0x0B: pC, // '\v' + 0x0C: pC, // '\f' + 0x0D: pC, // '\r' + 0x0E: pC, // '\x0e' + 0x0F: pC, // '\x0f' + 0x10: pC, // '\x10' + 0x11: pC, // '\x11' + 0x12: pC, // '\x12' + 0x13: pC, // '\x13' + 0x14: pC, // '\x14' + 0x15: pC, // '\x15' + 0x16: pC, // '\x16' + 0x17: pC, // '\x17' + 0x18: pC, // '\x18' + 0x19: pC, // '\x19' + 0x1A: pC, // '\x1a' + 0x1B: pC, // '\x1b' + 0x1C: pC, // '\x1c' + 0x1D: pC, // '\x1d' + 0x1E: pC, // '\x1e' + 0x1F: pC, // '\x1f' + 0x20: pZ | pp, // ' ' + 0x21: pP | pp, // '!' + 0x22: pP | pp, // '"' + 0x23: pP | pp, // '#' + 0x24: pS | pp, // '$' + 0x25: pP | pp, // '%' + 0x26: pP | pp, // '&' + 0x27: pP | pp, // '\'' + 0x28: pP | pp, // '(' + 0x29: pP | pp, // ')' + 0x2A: pP | pp, // '*' + 0x2B: pS | pp, // '+' + 0x2C: pP | pp, // ',' + 0x2D: pP | pp, // '-' + 0x2E: pP | pp, // '.' + 0x2F: pP | pp, // '/' + 0x30: pN | pp, // '0' + 0x31: pN | pp, // '1' + 0x32: pN | pp, // '2' + 0x33: pN | pp, // '3' + 0x34: pN | pp, // '4' + 0x35: pN | pp, // '5' + 0x36: pN | pp, // '6' + 0x37: pN | pp, // '7' + 0x38: pN | pp, // '8' + 0x39: pN | pp, // '9' + 0x3A: pP | pp, // ':' + 0x3B: pP | pp, // ';' + 0x3C: pS | pp, // '<' + 0x3D: pS | pp, // '=' + 0x3E: pS | pp, // '>' + 0x3F: pP | pp, // '?' + 0x40: pP | pp, // '@' + 0x41: pLu | pp, // 'A' + 0x42: pLu | pp, // 'B' + 0x43: pLu | pp, // 'C' + 0x44: pLu | pp, // 'D' + 0x45: pLu | pp, // 'E' + 0x46: pLu | pp, // 'F' + 0x47: pLu | pp, // 'G' + 0x48: pLu | pp, // 'H' + 0x49: pLu | pp, // 'I' + 0x4A: pLu | pp, // 'J' + 0x4B: pLu | pp, // 'K' + 0x4C: pLu | pp, // 'L' + 0x4D: pLu | pp, // 'M' + 0x4E: pLu | pp, // 'N' + 0x4F: pLu | pp, // 'O' + 0x50: pLu | pp, // 'P' + 0x51: pLu | pp, // 'Q' + 0x52: pLu | pp, // 'R' + 0x53: pLu | pp, // 'S' + 0x54: pLu | pp, // 'T' + 0x55: pLu | pp, // 'U' + 0x56: pLu | pp, // 'V' + 0x57: pLu | pp, // 'W' + 0x58: pLu | pp, // 'X' + 0x59: pLu | pp, // 'Y' + 0x5A: pLu | pp, // 'Z' + 0x5B: pP | pp, // '[' + 0x5C: pP | pp, // '\\' + 0x5D: pP | pp, // ']' + 0x5E: pS | pp, // '^' + 0x5F: pP | pp, // '_' + 0x60: pS | pp, // '`' + 0x61: pLl | pp, // 'a' + 0x62: pLl | pp, // 'b' + 0x63: pLl | pp, // 'c' + 0x64: pLl | pp, // 'd' + 0x65: pLl | pp, // 'e' + 0x66: pLl | pp, // 'f' + 0x67: pLl | pp, // 'g' + 0x68: pLl | pp, // 'h' + 0x69: pLl | pp, // 'i' + 0x6A: pLl | pp, // 'j' + 0x6B: pLl | pp, // 'k' + 0x6C: pLl | pp, // 'l' + 0x6D: pLl | pp, // 'm' + 0x6E: pLl | pp, // 'n' + 0x6F: pLl | pp, // 'o' + 0x70: pLl | pp, // 'p' + 0x71: pLl | pp, // 'q' + 0x72: pLl | pp, // 'r' + 0x73: pLl | pp, // 's' + 0x74: pLl | pp, // 't' + 0x75: pLl | pp, // 'u' + 0x76: pLl | pp, // 'v' + 0x77: pLl | pp, // 'w' + 0x78: pLl | pp, // 'x' + 0x79: pLl | pp, // 'y' + 0x7A: pLl | pp, // 'z' + 0x7B: pP | pp, // '{' + 0x7C: pS | pp, // '|' + 0x7D: pP | pp, // '}' + 0x7E: pS | pp, // '~' + 0x7F: pC, // '\x7f' + 0x80: pC, // '\u0080' + 0x81: pC, // '\u0081' + 0x82: pC, // '\u0082' + 0x83: pC, // '\u0083' + 0x84: pC, // '\u0084' + 0x85: pC, // '\u0085' + 0x86: pC, // '\u0086' + 0x87: pC, // '\u0087' + 0x88: pC, // '\u0088' + 0x89: pC, // '\u0089' + 0x8A: pC, // '\u008a' + 0x8B: pC, // '\u008b' + 0x8C: pC, // '\u008c' + 0x8D: pC, // '\u008d' + 0x8E: pC, // '\u008e' + 0x8F: pC, // '\u008f' + 0x90: pC, // '\u0090' + 0x91: pC, // '\u0091' + 0x92: pC, // '\u0092' + 0x93: pC, // '\u0093' + 0x94: pC, // '\u0094' + 0x95: pC, // '\u0095' + 0x96: pC, // '\u0096' + 0x97: pC, // '\u0097' + 0x98: pC, // '\u0098' + 0x99: pC, // '\u0099' + 0x9A: pC, // '\u009a' + 0x9B: pC, // '\u009b' + 0x9C: pC, // '\u009c' + 0x9D: pC, // '\u009d' + 0x9E: pC, // '\u009e' + 0x9F: pC, // '\u009f' + 0xA0: pZ, // '\u00a0' + 0xA1: pP | pp, // '\u00a1' + 0xA2: pS | pp, // '\u00a2' + 0xA3: pS | pp, // '\u00a3' + 0xA4: pS | pp, // '\u00a4' + 0xA5: pS | pp, // '\u00a5' + 0xA6: pS | pp, // '\u00a6' + 0xA7: pS | pp, // '\u00a7' + 0xA8: pS | pp, // '\u00a8' + 0xA9: pS | pp, // '\u00a9' + 0xAA: pLl | pp, // '\u00aa' + 0xAB: pP | pp, // '\u00ab' + 0xAC: pS | pp, // '\u00ac' + 0xAD: 0, // '\u00ad' + 0xAE: pS | pp, // '\u00ae' + 0xAF: pS | pp, // '\u00af' + 0xB0: pS | pp, // '\u00b0' + 0xB1: pS | pp, // '\u00b1' + 0xB2: pN | pp, // '\u00b2' + 0xB3: pN | pp, // '\u00b3' + 0xB4: pS | pp, // '\u00b4' + 0xB5: pLl | pp, // '\u00b5' + 0xB6: pS | pp, // '\u00b6' + 0xB7: pP | pp, // '\u00b7' + 0xB8: pS | pp, // '\u00b8' + 0xB9: pN | pp, // '\u00b9' + 0xBA: pLl | pp, // '\u00ba' + 0xBB: pP | pp, // '\u00bb' + 0xBC: pN | pp, // '\u00bc' + 0xBD: pN | pp, // '\u00bd' + 0xBE: pN | pp, // '\u00be' + 0xBF: pP | pp, // '\u00bf' + 0xC0: pLu | pp, // '\u00c0' + 0xC1: pLu | pp, // '\u00c1' + 0xC2: pLu | pp, // '\u00c2' + 0xC3: pLu | pp, // '\u00c3' + 0xC4: pLu | pp, // '\u00c4' + 0xC5: pLu | pp, // '\u00c5' + 0xC6: pLu | pp, // '\u00c6' + 0xC7: pLu | pp, // '\u00c7' + 0xC8: pLu | pp, // '\u00c8' + 0xC9: pLu | pp, // '\u00c9' + 0xCA: pLu | pp, // '\u00ca' + 0xCB: pLu | pp, // '\u00cb' + 0xCC: pLu | pp, // '\u00cc' + 0xCD: pLu | pp, // '\u00cd' + 0xCE: pLu | pp, // '\u00ce' + 0xCF: pLu | pp, // '\u00cf' + 0xD0: pLu | pp, // '\u00d0' + 0xD1: pLu | pp, // '\u00d1' + 0xD2: pLu | pp, // '\u00d2' + 0xD3: pLu | pp, // '\u00d3' + 0xD4: pLu | pp, // '\u00d4' + 0xD5: pLu | pp, // '\u00d5' + 0xD6: pLu | pp, // '\u00d6' + 0xD7: pS | pp, // '\u00d7' + 0xD8: pLu | pp, // '\u00d8' + 0xD9: pLu | pp, // '\u00d9' + 0xDA: pLu | pp, // '\u00da' + 0xDB: pLu | pp, // '\u00db' + 0xDC: pLu | pp, // '\u00dc' + 0xDD: pLu | pp, // '\u00dd' + 0xDE: pLu | pp, // '\u00de' + 0xDF: pLl | pp, // '\u00df' + 0xE0: pLl | pp, // '\u00e0' + 0xE1: pLl | pp, // '\u00e1' + 0xE2: pLl | pp, // '\u00e2' + 0xE3: pLl | pp, // '\u00e3' + 0xE4: pLl | pp, // '\u00e4' + 0xE5: pLl | pp, // '\u00e5' + 0xE6: pLl | pp, // '\u00e6' + 0xE7: pLl | pp, // '\u00e7' + 0xE8: pLl | pp, // '\u00e8' + 0xE9: pLl | pp, // '\u00e9' + 0xEA: pLl | pp, // '\u00ea' + 0xEB: pLl | pp, // '\u00eb' + 0xEC: pLl | pp, // '\u00ec' + 0xED: pLl | pp, // '\u00ed' + 0xEE: pLl | pp, // '\u00ee' + 0xEF: pLl | pp, // '\u00ef' + 0xF0: pLl | pp, // '\u00f0' + 0xF1: pLl | pp, // '\u00f1' + 0xF2: pLl | pp, // '\u00f2' + 0xF3: pLl | pp, // '\u00f3' + 0xF4: pLl | pp, // '\u00f4' + 0xF5: pLl | pp, // '\u00f5' + 0xF6: pLl | pp, // '\u00f6' + 0xF7: pS | pp, // '\u00f7' + 0xF8: pLl | pp, // '\u00f8' + 0xF9: pLl | pp, // '\u00f9' + 0xFA: pLl | pp, // '\u00fa' + 0xFB: pLl | pp, // '\u00fb' + 0xFC: pLl | pp, // '\u00fc' + 0xFD: pLl | pp, // '\u00fd' + 0xFE: pLl | pp, // '\u00fe' + 0xFF: pLl | pp, // '\u00ff' +} + +// Range entries: 3190 16-bit, 657 32-bit, 3847 total. +// Range bytes: 19140 16-bit, 7884 32-bit, 27024 total. diff --git a/src/pkg/websocket/client.go b/src/pkg/websocket/client.go index 78c8b7f57..e28382196 100644 --- a/src/pkg/websocket/client.go +++ b/src/pkg/websocket/client.go @@ -29,7 +29,7 @@ var ( ErrBadWebSocketOrigin = &ProtocolError{"missing or bad WebSocket-Origin"} ErrBadWebSocketLocation = &ProtocolError{"missing or bad WebSocket-Location"} ErrBadWebSocketProtocol = &ProtocolError{"missing or bad WebSocket-Protocol"} - ErrChallengeResponse = &ProtocolError{"mismatch challange/response"} + ErrChallengeResponse = &ProtocolError{"mismatch challenge/response"} secKeyRandomChars [0x30 - 0x21 + 0x7F - 0x3A]byte ) @@ -226,7 +226,7 @@ func handshake(resourceName, host, origin, location, protocol string, br *bufio. // Step 25. send CRLF. bw.WriteString("\r\n") - // Step 26. genearte 8 bytes random key. + // Step 26. generate 8 bytes random key. key3 := generateKey3() // Step 27. send it out. bw.Write(key3) @@ -235,7 +235,7 @@ func handshake(resourceName, host, origin, location, protocol string, br *bufio. } // Step 28-29, 32-40. read response from server. - resp, err := http.ReadResponse(br, "GET") + resp, err := http.ReadResponse(br, &http.Request{Method: "GET"}) if err != nil { return err } @@ -262,7 +262,7 @@ func handshake(resourceName, host, origin, location, protocol string, br *bufio. return ErrBadWebSocketProtocol } - // Step 42-43. get expected data from challange data. + // Step 42-43. get expected data from challenge data. expected, err := getChallengeResponse(number1, number2, key3) if err != nil { return err @@ -283,7 +283,7 @@ func handshake(resourceName, host, origin, location, protocol string, br *bufio. } /* -Handhake described in (soon obsolete) +Handshake described in (soon obsolete) draft-hixie-thewebsocket-protocol-75. */ func draft75handshake(resourceName, host, origin, location, protocol string, br *bufio.Reader, bw *bufio.Writer) (err os.Error) { @@ -297,7 +297,7 @@ func draft75handshake(resourceName, host, origin, location, protocol string, br } bw.WriteString("\r\n") bw.Flush() - resp, err := http.ReadResponse(br, "GET") + resp, err := http.ReadResponse(br, &http.Request{Method: "GET"}) if err != nil { return } diff --git a/src/pkg/websocket/server.go b/src/pkg/websocket/server.go index 376265236..165cbffee 100644 --- a/src/pkg/websocket/server.go +++ b/src/pkg/websocket/server.go @@ -124,7 +124,7 @@ func (f Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { part1 := keyNumber1 / space1 part2 := keyNumber2 / space2 - // Step 8. let challenge to be concatination of part1, part2 and key3. + // Step 8. let challenge be concatenation of part1, part2 and key3. // Step 9. get MD5 fingerprint of challenge. response, err := getChallengeResponse(part1, part2, key3) if err != nil { diff --git a/src/pkg/websocket/websocket.go b/src/pkg/websocket/websocket.go index edde61b4a..7447cf852 100644 --- a/src/pkg/websocket/websocket.go +++ b/src/pkg/websocket/websocket.go @@ -158,7 +158,7 @@ func (ws *Conn) SetReadTimeout(nsec int64) os.Error { return os.EINVAL } -// SetWritetTimeout sets the connection's network write timeout in nanoseconds. +// SetWriteTimeout sets the connection's network write timeout in nanoseconds. func (ws *Conn) SetWriteTimeout(nsec int64) os.Error { if conn, ok := ws.rwc.(net.Conn); ok { return conn.SetWriteTimeout(nsec) diff --git a/src/pkg/websocket/websocket_test.go b/src/pkg/websocket/websocket_test.go index 10f88dfd1..84788b416 100644 --- a/src/pkg/websocket/websocket_test.go +++ b/src/pkg/websocket/websocket_test.go @@ -150,7 +150,7 @@ func TestHTTP(t *testing.T) { // If the client did not send a handshake that matches the protocol // specification, the server should abort the WebSocket connection. - _, _, err := http.Get(fmt.Sprintf("http://%s/echo", serverAddr)) + _, err := http.Get(fmt.Sprintf("http://%s/echo", serverAddr)) if err == nil { t.Error("Get: unexpected success") return @@ -169,7 +169,7 @@ func TestHTTP(t *testing.T) { func TestHTTPDraft75(t *testing.T) { once.Do(startServer) - r, _, err := http.Get(fmt.Sprintf("http://%s/echoDraft75", serverAddr)) + r, err := http.Get(fmt.Sprintf("http://%s/echoDraft75", serverAddr)) if err != nil { t.Errorf("Get: error %#v", err) return diff --git a/src/pkg/xml/xml.go b/src/pkg/xml/xml.go index 42d8b986e..2cebbce75 100644 --- a/src/pkg/xml/xml.go +++ b/src/pkg/xml/xml.go @@ -1028,312 +1028,316 @@ func isNameByte(c byte) bool { // and then reformatting. First corresponds to (Letter | '_' | ':') // and second corresponds to NameChar. -var first = []unicode.Range{ - {0x003A, 0x003A, 1}, - {0x0041, 0x005A, 1}, - {0x005F, 0x005F, 1}, - {0x0061, 0x007A, 1}, - {0x00C0, 0x00D6, 1}, - {0x00D8, 0x00F6, 1}, - {0x00F8, 0x00FF, 1}, - {0x0100, 0x0131, 1}, - {0x0134, 0x013E, 1}, - {0x0141, 0x0148, 1}, - {0x014A, 0x017E, 1}, - {0x0180, 0x01C3, 1}, - {0x01CD, 0x01F0, 1}, - {0x01F4, 0x01F5, 1}, - {0x01FA, 0x0217, 1}, - {0x0250, 0x02A8, 1}, - {0x02BB, 0x02C1, 1}, - {0x0386, 0x0386, 1}, - {0x0388, 0x038A, 1}, - {0x038C, 0x038C, 1}, - {0x038E, 0x03A1, 1}, - {0x03A3, 0x03CE, 1}, - {0x03D0, 0x03D6, 1}, - {0x03DA, 0x03E0, 2}, - {0x03E2, 0x03F3, 1}, - {0x0401, 0x040C, 1}, - {0x040E, 0x044F, 1}, - {0x0451, 0x045C, 1}, - {0x045E, 0x0481, 1}, - {0x0490, 0x04C4, 1}, - {0x04C7, 0x04C8, 1}, - {0x04CB, 0x04CC, 1}, - {0x04D0, 0x04EB, 1}, - {0x04EE, 0x04F5, 1}, - {0x04F8, 0x04F9, 1}, - {0x0531, 0x0556, 1}, - {0x0559, 0x0559, 1}, - {0x0561, 0x0586, 1}, - {0x05D0, 0x05EA, 1}, - {0x05F0, 0x05F2, 1}, - {0x0621, 0x063A, 1}, - {0x0641, 0x064A, 1}, - {0x0671, 0x06B7, 1}, - {0x06BA, 0x06BE, 1}, - {0x06C0, 0x06CE, 1}, - {0x06D0, 0x06D3, 1}, - {0x06D5, 0x06D5, 1}, - {0x06E5, 0x06E6, 1}, - {0x0905, 0x0939, 1}, - {0x093D, 0x093D, 1}, - {0x0958, 0x0961, 1}, - {0x0985, 0x098C, 1}, - {0x098F, 0x0990, 1}, - {0x0993, 0x09A8, 1}, - {0x09AA, 0x09B0, 1}, - {0x09B2, 0x09B2, 1}, - {0x09B6, 0x09B9, 1}, - {0x09DC, 0x09DD, 1}, - {0x09DF, 0x09E1, 1}, - {0x09F0, 0x09F1, 1}, - {0x0A05, 0x0A0A, 1}, - {0x0A0F, 0x0A10, 1}, - {0x0A13, 0x0A28, 1}, - {0x0A2A, 0x0A30, 1}, - {0x0A32, 0x0A33, 1}, - {0x0A35, 0x0A36, 1}, - {0x0A38, 0x0A39, 1}, - {0x0A59, 0x0A5C, 1}, - {0x0A5E, 0x0A5E, 1}, - {0x0A72, 0x0A74, 1}, - {0x0A85, 0x0A8B, 1}, - {0x0A8D, 0x0A8D, 1}, - {0x0A8F, 0x0A91, 1}, - {0x0A93, 0x0AA8, 1}, - {0x0AAA, 0x0AB0, 1}, - {0x0AB2, 0x0AB3, 1}, - {0x0AB5, 0x0AB9, 1}, - {0x0ABD, 0x0AE0, 0x23}, - {0x0B05, 0x0B0C, 1}, - {0x0B0F, 0x0B10, 1}, - {0x0B13, 0x0B28, 1}, - {0x0B2A, 0x0B30, 1}, - {0x0B32, 0x0B33, 1}, - {0x0B36, 0x0B39, 1}, - {0x0B3D, 0x0B3D, 1}, - {0x0B5C, 0x0B5D, 1}, - {0x0B5F, 0x0B61, 1}, - {0x0B85, 0x0B8A, 1}, - {0x0B8E, 0x0B90, 1}, - {0x0B92, 0x0B95, 1}, - {0x0B99, 0x0B9A, 1}, - {0x0B9C, 0x0B9C, 1}, - {0x0B9E, 0x0B9F, 1}, - {0x0BA3, 0x0BA4, 1}, - {0x0BA8, 0x0BAA, 1}, - {0x0BAE, 0x0BB5, 1}, - {0x0BB7, 0x0BB9, 1}, - {0x0C05, 0x0C0C, 1}, - {0x0C0E, 0x0C10, 1}, - {0x0C12, 0x0C28, 1}, - {0x0C2A, 0x0C33, 1}, - {0x0C35, 0x0C39, 1}, - {0x0C60, 0x0C61, 1}, - {0x0C85, 0x0C8C, 1}, - {0x0C8E, 0x0C90, 1}, - {0x0C92, 0x0CA8, 1}, - {0x0CAA, 0x0CB3, 1}, - {0x0CB5, 0x0CB9, 1}, - {0x0CDE, 0x0CDE, 1}, - {0x0CE0, 0x0CE1, 1}, - {0x0D05, 0x0D0C, 1}, - {0x0D0E, 0x0D10, 1}, - {0x0D12, 0x0D28, 1}, - {0x0D2A, 0x0D39, 1}, - {0x0D60, 0x0D61, 1}, - {0x0E01, 0x0E2E, 1}, - {0x0E30, 0x0E30, 1}, - {0x0E32, 0x0E33, 1}, - {0x0E40, 0x0E45, 1}, - {0x0E81, 0x0E82, 1}, - {0x0E84, 0x0E84, 1}, - {0x0E87, 0x0E88, 1}, - {0x0E8A, 0x0E8D, 3}, - {0x0E94, 0x0E97, 1}, - {0x0E99, 0x0E9F, 1}, - {0x0EA1, 0x0EA3, 1}, - {0x0EA5, 0x0EA7, 2}, - {0x0EAA, 0x0EAB, 1}, - {0x0EAD, 0x0EAE, 1}, - {0x0EB0, 0x0EB0, 1}, - {0x0EB2, 0x0EB3, 1}, - {0x0EBD, 0x0EBD, 1}, - {0x0EC0, 0x0EC4, 1}, - {0x0F40, 0x0F47, 1}, - {0x0F49, 0x0F69, 1}, - {0x10A0, 0x10C5, 1}, - {0x10D0, 0x10F6, 1}, - {0x1100, 0x1100, 1}, - {0x1102, 0x1103, 1}, - {0x1105, 0x1107, 1}, - {0x1109, 0x1109, 1}, - {0x110B, 0x110C, 1}, - {0x110E, 0x1112, 1}, - {0x113C, 0x1140, 2}, - {0x114C, 0x1150, 2}, - {0x1154, 0x1155, 1}, - {0x1159, 0x1159, 1}, - {0x115F, 0x1161, 1}, - {0x1163, 0x1169, 2}, - {0x116D, 0x116E, 1}, - {0x1172, 0x1173, 1}, - {0x1175, 0x119E, 0x119E - 0x1175}, - {0x11A8, 0x11AB, 0x11AB - 0x11A8}, - {0x11AE, 0x11AF, 1}, - {0x11B7, 0x11B8, 1}, - {0x11BA, 0x11BA, 1}, - {0x11BC, 0x11C2, 1}, - {0x11EB, 0x11F0, 0x11F0 - 0x11EB}, - {0x11F9, 0x11F9, 1}, - {0x1E00, 0x1E9B, 1}, - {0x1EA0, 0x1EF9, 1}, - {0x1F00, 0x1F15, 1}, - {0x1F18, 0x1F1D, 1}, - {0x1F20, 0x1F45, 1}, - {0x1F48, 0x1F4D, 1}, - {0x1F50, 0x1F57, 1}, - {0x1F59, 0x1F5B, 0x1F5B - 0x1F59}, - {0x1F5D, 0x1F5D, 1}, - {0x1F5F, 0x1F7D, 1}, - {0x1F80, 0x1FB4, 1}, - {0x1FB6, 0x1FBC, 1}, - {0x1FBE, 0x1FBE, 1}, - {0x1FC2, 0x1FC4, 1}, - {0x1FC6, 0x1FCC, 1}, - {0x1FD0, 0x1FD3, 1}, - {0x1FD6, 0x1FDB, 1}, - {0x1FE0, 0x1FEC, 1}, - {0x1FF2, 0x1FF4, 1}, - {0x1FF6, 0x1FFC, 1}, - {0x2126, 0x2126, 1}, - {0x212A, 0x212B, 1}, - {0x212E, 0x212E, 1}, - {0x2180, 0x2182, 1}, - {0x3007, 0x3007, 1}, - {0x3021, 0x3029, 1}, - {0x3041, 0x3094, 1}, - {0x30A1, 0x30FA, 1}, - {0x3105, 0x312C, 1}, - {0x4E00, 0x9FA5, 1}, - {0xAC00, 0xD7A3, 1}, +var first = &unicode.RangeTable{ + R16: []unicode.Range16{ + {0x003A, 0x003A, 1}, + {0x0041, 0x005A, 1}, + {0x005F, 0x005F, 1}, + {0x0061, 0x007A, 1}, + {0x00C0, 0x00D6, 1}, + {0x00D8, 0x00F6, 1}, + {0x00F8, 0x00FF, 1}, + {0x0100, 0x0131, 1}, + {0x0134, 0x013E, 1}, + {0x0141, 0x0148, 1}, + {0x014A, 0x017E, 1}, + {0x0180, 0x01C3, 1}, + {0x01CD, 0x01F0, 1}, + {0x01F4, 0x01F5, 1}, + {0x01FA, 0x0217, 1}, + {0x0250, 0x02A8, 1}, + {0x02BB, 0x02C1, 1}, + {0x0386, 0x0386, 1}, + {0x0388, 0x038A, 1}, + {0x038C, 0x038C, 1}, + {0x038E, 0x03A1, 1}, + {0x03A3, 0x03CE, 1}, + {0x03D0, 0x03D6, 1}, + {0x03DA, 0x03E0, 2}, + {0x03E2, 0x03F3, 1}, + {0x0401, 0x040C, 1}, + {0x040E, 0x044F, 1}, + {0x0451, 0x045C, 1}, + {0x045E, 0x0481, 1}, + {0x0490, 0x04C4, 1}, + {0x04C7, 0x04C8, 1}, + {0x04CB, 0x04CC, 1}, + {0x04D0, 0x04EB, 1}, + {0x04EE, 0x04F5, 1}, + {0x04F8, 0x04F9, 1}, + {0x0531, 0x0556, 1}, + {0x0559, 0x0559, 1}, + {0x0561, 0x0586, 1}, + {0x05D0, 0x05EA, 1}, + {0x05F0, 0x05F2, 1}, + {0x0621, 0x063A, 1}, + {0x0641, 0x064A, 1}, + {0x0671, 0x06B7, 1}, + {0x06BA, 0x06BE, 1}, + {0x06C0, 0x06CE, 1}, + {0x06D0, 0x06D3, 1}, + {0x06D5, 0x06D5, 1}, + {0x06E5, 0x06E6, 1}, + {0x0905, 0x0939, 1}, + {0x093D, 0x093D, 1}, + {0x0958, 0x0961, 1}, + {0x0985, 0x098C, 1}, + {0x098F, 0x0990, 1}, + {0x0993, 0x09A8, 1}, + {0x09AA, 0x09B0, 1}, + {0x09B2, 0x09B2, 1}, + {0x09B6, 0x09B9, 1}, + {0x09DC, 0x09DD, 1}, + {0x09DF, 0x09E1, 1}, + {0x09F0, 0x09F1, 1}, + {0x0A05, 0x0A0A, 1}, + {0x0A0F, 0x0A10, 1}, + {0x0A13, 0x0A28, 1}, + {0x0A2A, 0x0A30, 1}, + {0x0A32, 0x0A33, 1}, + {0x0A35, 0x0A36, 1}, + {0x0A38, 0x0A39, 1}, + {0x0A59, 0x0A5C, 1}, + {0x0A5E, 0x0A5E, 1}, + {0x0A72, 0x0A74, 1}, + {0x0A85, 0x0A8B, 1}, + {0x0A8D, 0x0A8D, 1}, + {0x0A8F, 0x0A91, 1}, + {0x0A93, 0x0AA8, 1}, + {0x0AAA, 0x0AB0, 1}, + {0x0AB2, 0x0AB3, 1}, + {0x0AB5, 0x0AB9, 1}, + {0x0ABD, 0x0AE0, 0x23}, + {0x0B05, 0x0B0C, 1}, + {0x0B0F, 0x0B10, 1}, + {0x0B13, 0x0B28, 1}, + {0x0B2A, 0x0B30, 1}, + {0x0B32, 0x0B33, 1}, + {0x0B36, 0x0B39, 1}, + {0x0B3D, 0x0B3D, 1}, + {0x0B5C, 0x0B5D, 1}, + {0x0B5F, 0x0B61, 1}, + {0x0B85, 0x0B8A, 1}, + {0x0B8E, 0x0B90, 1}, + {0x0B92, 0x0B95, 1}, + {0x0B99, 0x0B9A, 1}, + {0x0B9C, 0x0B9C, 1}, + {0x0B9E, 0x0B9F, 1}, + {0x0BA3, 0x0BA4, 1}, + {0x0BA8, 0x0BAA, 1}, + {0x0BAE, 0x0BB5, 1}, + {0x0BB7, 0x0BB9, 1}, + {0x0C05, 0x0C0C, 1}, + {0x0C0E, 0x0C10, 1}, + {0x0C12, 0x0C28, 1}, + {0x0C2A, 0x0C33, 1}, + {0x0C35, 0x0C39, 1}, + {0x0C60, 0x0C61, 1}, + {0x0C85, 0x0C8C, 1}, + {0x0C8E, 0x0C90, 1}, + {0x0C92, 0x0CA8, 1}, + {0x0CAA, 0x0CB3, 1}, + {0x0CB5, 0x0CB9, 1}, + {0x0CDE, 0x0CDE, 1}, + {0x0CE0, 0x0CE1, 1}, + {0x0D05, 0x0D0C, 1}, + {0x0D0E, 0x0D10, 1}, + {0x0D12, 0x0D28, 1}, + {0x0D2A, 0x0D39, 1}, + {0x0D60, 0x0D61, 1}, + {0x0E01, 0x0E2E, 1}, + {0x0E30, 0x0E30, 1}, + {0x0E32, 0x0E33, 1}, + {0x0E40, 0x0E45, 1}, + {0x0E81, 0x0E82, 1}, + {0x0E84, 0x0E84, 1}, + {0x0E87, 0x0E88, 1}, + {0x0E8A, 0x0E8D, 3}, + {0x0E94, 0x0E97, 1}, + {0x0E99, 0x0E9F, 1}, + {0x0EA1, 0x0EA3, 1}, + {0x0EA5, 0x0EA7, 2}, + {0x0EAA, 0x0EAB, 1}, + {0x0EAD, 0x0EAE, 1}, + {0x0EB0, 0x0EB0, 1}, + {0x0EB2, 0x0EB3, 1}, + {0x0EBD, 0x0EBD, 1}, + {0x0EC0, 0x0EC4, 1}, + {0x0F40, 0x0F47, 1}, + {0x0F49, 0x0F69, 1}, + {0x10A0, 0x10C5, 1}, + {0x10D0, 0x10F6, 1}, + {0x1100, 0x1100, 1}, + {0x1102, 0x1103, 1}, + {0x1105, 0x1107, 1}, + {0x1109, 0x1109, 1}, + {0x110B, 0x110C, 1}, + {0x110E, 0x1112, 1}, + {0x113C, 0x1140, 2}, + {0x114C, 0x1150, 2}, + {0x1154, 0x1155, 1}, + {0x1159, 0x1159, 1}, + {0x115F, 0x1161, 1}, + {0x1163, 0x1169, 2}, + {0x116D, 0x116E, 1}, + {0x1172, 0x1173, 1}, + {0x1175, 0x119E, 0x119E - 0x1175}, + {0x11A8, 0x11AB, 0x11AB - 0x11A8}, + {0x11AE, 0x11AF, 1}, + {0x11B7, 0x11B8, 1}, + {0x11BA, 0x11BA, 1}, + {0x11BC, 0x11C2, 1}, + {0x11EB, 0x11F0, 0x11F0 - 0x11EB}, + {0x11F9, 0x11F9, 1}, + {0x1E00, 0x1E9B, 1}, + {0x1EA0, 0x1EF9, 1}, + {0x1F00, 0x1F15, 1}, + {0x1F18, 0x1F1D, 1}, + {0x1F20, 0x1F45, 1}, + {0x1F48, 0x1F4D, 1}, + {0x1F50, 0x1F57, 1}, + {0x1F59, 0x1F5B, 0x1F5B - 0x1F59}, + {0x1F5D, 0x1F5D, 1}, + {0x1F5F, 0x1F7D, 1}, + {0x1F80, 0x1FB4, 1}, + {0x1FB6, 0x1FBC, 1}, + {0x1FBE, 0x1FBE, 1}, + {0x1FC2, 0x1FC4, 1}, + {0x1FC6, 0x1FCC, 1}, + {0x1FD0, 0x1FD3, 1}, + {0x1FD6, 0x1FDB, 1}, + {0x1FE0, 0x1FEC, 1}, + {0x1FF2, 0x1FF4, 1}, + {0x1FF6, 0x1FFC, 1}, + {0x2126, 0x2126, 1}, + {0x212A, 0x212B, 1}, + {0x212E, 0x212E, 1}, + {0x2180, 0x2182, 1}, + {0x3007, 0x3007, 1}, + {0x3021, 0x3029, 1}, + {0x3041, 0x3094, 1}, + {0x30A1, 0x30FA, 1}, + {0x3105, 0x312C, 1}, + {0x4E00, 0x9FA5, 1}, + {0xAC00, 0xD7A3, 1}, + }, } -var second = []unicode.Range{ - {0x002D, 0x002E, 1}, - {0x0030, 0x0039, 1}, - {0x00B7, 0x00B7, 1}, - {0x02D0, 0x02D1, 1}, - {0x0300, 0x0345, 1}, - {0x0360, 0x0361, 1}, - {0x0387, 0x0387, 1}, - {0x0483, 0x0486, 1}, - {0x0591, 0x05A1, 1}, - {0x05A3, 0x05B9, 1}, - {0x05BB, 0x05BD, 1}, - {0x05BF, 0x05BF, 1}, - {0x05C1, 0x05C2, 1}, - {0x05C4, 0x0640, 0x0640 - 0x05C4}, - {0x064B, 0x0652, 1}, - {0x0660, 0x0669, 1}, - {0x0670, 0x0670, 1}, - {0x06D6, 0x06DC, 1}, - {0x06DD, 0x06DF, 1}, - {0x06E0, 0x06E4, 1}, - {0x06E7, 0x06E8, 1}, - {0x06EA, 0x06ED, 1}, - {0x06F0, 0x06F9, 1}, - {0x0901, 0x0903, 1}, - {0x093C, 0x093C, 1}, - {0x093E, 0x094C, 1}, - {0x094D, 0x094D, 1}, - {0x0951, 0x0954, 1}, - {0x0962, 0x0963, 1}, - {0x0966, 0x096F, 1}, - {0x0981, 0x0983, 1}, - {0x09BC, 0x09BC, 1}, - {0x09BE, 0x09BF, 1}, - {0x09C0, 0x09C4, 1}, - {0x09C7, 0x09C8, 1}, - {0x09CB, 0x09CD, 1}, - {0x09D7, 0x09D7, 1}, - {0x09E2, 0x09E3, 1}, - {0x09E6, 0x09EF, 1}, - {0x0A02, 0x0A3C, 0x3A}, - {0x0A3E, 0x0A3F, 1}, - {0x0A40, 0x0A42, 1}, - {0x0A47, 0x0A48, 1}, - {0x0A4B, 0x0A4D, 1}, - {0x0A66, 0x0A6F, 1}, - {0x0A70, 0x0A71, 1}, - {0x0A81, 0x0A83, 1}, - {0x0ABC, 0x0ABC, 1}, - {0x0ABE, 0x0AC5, 1}, - {0x0AC7, 0x0AC9, 1}, - {0x0ACB, 0x0ACD, 1}, - {0x0AE6, 0x0AEF, 1}, - {0x0B01, 0x0B03, 1}, - {0x0B3C, 0x0B3C, 1}, - {0x0B3E, 0x0B43, 1}, - {0x0B47, 0x0B48, 1}, - {0x0B4B, 0x0B4D, 1}, - {0x0B56, 0x0B57, 1}, - {0x0B66, 0x0B6F, 1}, - {0x0B82, 0x0B83, 1}, - {0x0BBE, 0x0BC2, 1}, - {0x0BC6, 0x0BC8, 1}, - {0x0BCA, 0x0BCD, 1}, - {0x0BD7, 0x0BD7, 1}, - {0x0BE7, 0x0BEF, 1}, - {0x0C01, 0x0C03, 1}, - {0x0C3E, 0x0C44, 1}, - {0x0C46, 0x0C48, 1}, - {0x0C4A, 0x0C4D, 1}, - {0x0C55, 0x0C56, 1}, - {0x0C66, 0x0C6F, 1}, - {0x0C82, 0x0C83, 1}, - {0x0CBE, 0x0CC4, 1}, - {0x0CC6, 0x0CC8, 1}, - {0x0CCA, 0x0CCD, 1}, - {0x0CD5, 0x0CD6, 1}, - {0x0CE6, 0x0CEF, 1}, - {0x0D02, 0x0D03, 1}, - {0x0D3E, 0x0D43, 1}, - {0x0D46, 0x0D48, 1}, - {0x0D4A, 0x0D4D, 1}, - {0x0D57, 0x0D57, 1}, - {0x0D66, 0x0D6F, 1}, - {0x0E31, 0x0E31, 1}, - {0x0E34, 0x0E3A, 1}, - {0x0E46, 0x0E46, 1}, - {0x0E47, 0x0E4E, 1}, - {0x0E50, 0x0E59, 1}, - {0x0EB1, 0x0EB1, 1}, - {0x0EB4, 0x0EB9, 1}, - {0x0EBB, 0x0EBC, 1}, - {0x0EC6, 0x0EC6, 1}, - {0x0EC8, 0x0ECD, 1}, - {0x0ED0, 0x0ED9, 1}, - {0x0F18, 0x0F19, 1}, - {0x0F20, 0x0F29, 1}, - {0x0F35, 0x0F39, 2}, - {0x0F3E, 0x0F3F, 1}, - {0x0F71, 0x0F84, 1}, - {0x0F86, 0x0F8B, 1}, - {0x0F90, 0x0F95, 1}, - {0x0F97, 0x0F97, 1}, - {0x0F99, 0x0FAD, 1}, - {0x0FB1, 0x0FB7, 1}, - {0x0FB9, 0x0FB9, 1}, - {0x20D0, 0x20DC, 1}, - {0x20E1, 0x3005, 0x3005 - 0x20E1}, - {0x302A, 0x302F, 1}, - {0x3031, 0x3035, 1}, - {0x3099, 0x309A, 1}, - {0x309D, 0x309E, 1}, - {0x30FC, 0x30FE, 1}, +var second = &unicode.RangeTable{ + R16: []unicode.Range16{ + {0x002D, 0x002E, 1}, + {0x0030, 0x0039, 1}, + {0x00B7, 0x00B7, 1}, + {0x02D0, 0x02D1, 1}, + {0x0300, 0x0345, 1}, + {0x0360, 0x0361, 1}, + {0x0387, 0x0387, 1}, + {0x0483, 0x0486, 1}, + {0x0591, 0x05A1, 1}, + {0x05A3, 0x05B9, 1}, + {0x05BB, 0x05BD, 1}, + {0x05BF, 0x05BF, 1}, + {0x05C1, 0x05C2, 1}, + {0x05C4, 0x0640, 0x0640 - 0x05C4}, + {0x064B, 0x0652, 1}, + {0x0660, 0x0669, 1}, + {0x0670, 0x0670, 1}, + {0x06D6, 0x06DC, 1}, + {0x06DD, 0x06DF, 1}, + {0x06E0, 0x06E4, 1}, + {0x06E7, 0x06E8, 1}, + {0x06EA, 0x06ED, 1}, + {0x06F0, 0x06F9, 1}, + {0x0901, 0x0903, 1}, + {0x093C, 0x093C, 1}, + {0x093E, 0x094C, 1}, + {0x094D, 0x094D, 1}, + {0x0951, 0x0954, 1}, + {0x0962, 0x0963, 1}, + {0x0966, 0x096F, 1}, + {0x0981, 0x0983, 1}, + {0x09BC, 0x09BC, 1}, + {0x09BE, 0x09BF, 1}, + {0x09C0, 0x09C4, 1}, + {0x09C7, 0x09C8, 1}, + {0x09CB, 0x09CD, 1}, + {0x09D7, 0x09D7, 1}, + {0x09E2, 0x09E3, 1}, + {0x09E6, 0x09EF, 1}, + {0x0A02, 0x0A3C, 0x3A}, + {0x0A3E, 0x0A3F, 1}, + {0x0A40, 0x0A42, 1}, + {0x0A47, 0x0A48, 1}, + {0x0A4B, 0x0A4D, 1}, + {0x0A66, 0x0A6F, 1}, + {0x0A70, 0x0A71, 1}, + {0x0A81, 0x0A83, 1}, + {0x0ABC, 0x0ABC, 1}, + {0x0ABE, 0x0AC5, 1}, + {0x0AC7, 0x0AC9, 1}, + {0x0ACB, 0x0ACD, 1}, + {0x0AE6, 0x0AEF, 1}, + {0x0B01, 0x0B03, 1}, + {0x0B3C, 0x0B3C, 1}, + {0x0B3E, 0x0B43, 1}, + {0x0B47, 0x0B48, 1}, + {0x0B4B, 0x0B4D, 1}, + {0x0B56, 0x0B57, 1}, + {0x0B66, 0x0B6F, 1}, + {0x0B82, 0x0B83, 1}, + {0x0BBE, 0x0BC2, 1}, + {0x0BC6, 0x0BC8, 1}, + {0x0BCA, 0x0BCD, 1}, + {0x0BD7, 0x0BD7, 1}, + {0x0BE7, 0x0BEF, 1}, + {0x0C01, 0x0C03, 1}, + {0x0C3E, 0x0C44, 1}, + {0x0C46, 0x0C48, 1}, + {0x0C4A, 0x0C4D, 1}, + {0x0C55, 0x0C56, 1}, + {0x0C66, 0x0C6F, 1}, + {0x0C82, 0x0C83, 1}, + {0x0CBE, 0x0CC4, 1}, + {0x0CC6, 0x0CC8, 1}, + {0x0CCA, 0x0CCD, 1}, + {0x0CD5, 0x0CD6, 1}, + {0x0CE6, 0x0CEF, 1}, + {0x0D02, 0x0D03, 1}, + {0x0D3E, 0x0D43, 1}, + {0x0D46, 0x0D48, 1}, + {0x0D4A, 0x0D4D, 1}, + {0x0D57, 0x0D57, 1}, + {0x0D66, 0x0D6F, 1}, + {0x0E31, 0x0E31, 1}, + {0x0E34, 0x0E3A, 1}, + {0x0E46, 0x0E46, 1}, + {0x0E47, 0x0E4E, 1}, + {0x0E50, 0x0E59, 1}, + {0x0EB1, 0x0EB1, 1}, + {0x0EB4, 0x0EB9, 1}, + {0x0EBB, 0x0EBC, 1}, + {0x0EC6, 0x0EC6, 1}, + {0x0EC8, 0x0ECD, 1}, + {0x0ED0, 0x0ED9, 1}, + {0x0F18, 0x0F19, 1}, + {0x0F20, 0x0F29, 1}, + {0x0F35, 0x0F39, 2}, + {0x0F3E, 0x0F3F, 1}, + {0x0F71, 0x0F84, 1}, + {0x0F86, 0x0F8B, 1}, + {0x0F90, 0x0F95, 1}, + {0x0F97, 0x0F97, 1}, + {0x0F99, 0x0FAD, 1}, + {0x0FB1, 0x0FB7, 1}, + {0x0FB9, 0x0FB9, 1}, + {0x20D0, 0x20DC, 1}, + {0x20E1, 0x3005, 0x3005 - 0x20E1}, + {0x302A, 0x302F, 1}, + {0x3031, 0x3035, 1}, + {0x3099, 0x309A, 1}, + {0x309D, 0x309E, 1}, + {0x30FC, 0x30FE, 1}, + }, } // HTMLEntity is an entity map containing translations for the diff --git a/src/quietgcc.bash b/src/quietgcc.bash index e29ee4f64..c99305eda 100755 --- a/src/quietgcc.bash +++ b/src/quietgcc.bash @@ -4,7 +4,7 @@ # license that can be found in the LICENSE file. # The master for this file is $GOROOT/src/quietgcc.bash -# Changes made to $GOBIN/quietgcc will be overridden. +# Changes made to $GOBIN/quietgcc will be overwritten. # Gcc output that we don't care to see. ignore=': error: .Each undeclared identifier' @@ -32,7 +32,7 @@ case "$(uname -m -p)-$GOHOSTARCH" in esac # Run gcc, save error status, redisplay output without noise, exit with gcc status. -tmp=/tmp/qcc.$$.$USER.out +tmp=${TMPDIR:-/tmp}/quietgcc.$$.$USER.out $gcc -Wall -Wno-sign-compare -Wno-missing-braces \ -Wno-parentheses -Wno-unknown-pragmas -Wno-switch -Wno-comment \ -Werror \ diff --git a/test/bench/timing.log b/test/bench/timing.log index c658fa098..e66797982 100644 --- a/test/bench/timing.log +++ b/test/bench/timing.log @@ -844,3 +844,11 @@ NEW: gcc -O2 chameneosredux.c -lpthread 8.05u 63.43s 11.16r gccgo -O2 chameneosredux.go 82.95u 304.37s 207.64r gc chameneosredux 9.42u 0.00s 9.43r + +# May 13, 2011 +# after gc update to inline append when possible - 35% faster + +regex-dna 100000 + gc regex-dna 3.94u 0.00s 3.95r + gc regex-dna-parallel 4.15u 0.01s 1.63r + gc_B regex-dna 4.01u 0.01s 4.02r diff --git a/test/ddd1.go b/test/ddd1.go index a0bc73814..96a358e1c 100644 --- a/test/ddd1.go +++ b/test/ddd1.go @@ -43,4 +43,5 @@ func bad(args ...int) { var x int _ = unsafe.Pointer(&x...) // ERROR "[.][.][.]" _ = unsafe.Sizeof(x...) // ERROR "[.][.][.]" + _ = [...]byte("foo") // ERROR "[.][.][.]" } diff --git a/test/fixedbugs/bug226.dir/y.go b/test/fixedbugs/bug226.dir/y.go index 01e8b7b43..c66d592b7 100644 --- a/test/fixedbugs/bug226.dir/y.go +++ b/test/fixedbugs/bug226.dir/y.go @@ -15,7 +15,7 @@ func f() { _ = x.T{}; _ = x.T{Y:2}; - ok1.M(); // ERROR "assignment.*T" + ok1.M(); bad1 := *ok; // ERROR "assignment.*T" bad2 := ok1; // ERROR "assignment.*T" *ok4 = ok1; // ERROR "assignment.*T" diff --git a/test/bugs/bug322.dir/lib.go b/test/fixedbugs/bug322.dir/lib.go index 0de56d3d6..0de56d3d6 100644 --- a/test/bugs/bug322.dir/lib.go +++ b/test/fixedbugs/bug322.dir/lib.go diff --git a/test/bugs/bug322.dir/main.go b/test/fixedbugs/bug322.dir/main.go index 0ab5b32e4..f403c7d32 100644 --- a/test/bugs/bug322.dir/main.go +++ b/test/fixedbugs/bug322.dir/main.go @@ -38,10 +38,3 @@ func main() { var pi2 PI = pt pi2.PM() } - -/* -These should not be errors anymore: - -bug322.dir/main.go:19: implicit assignment of unexported field 'x' of lib.T in method receiver -bug322.dir/main.go:32: implicit assignment of unexported field 'x' of lib.T in method receiver -*/ diff --git a/test/bugs/bug322.go b/test/fixedbugs/bug322.go index ad0e62dc8..ad0e62dc8 100644 --- a/test/bugs/bug322.go +++ b/test/fixedbugs/bug322.go diff --git a/test/fixedbugs/bug337.go b/test/fixedbugs/bug337.go new file mode 100644 index 000000000..62e310e72 --- /dev/null +++ b/test/fixedbugs/bug337.go @@ -0,0 +1,19 @@ +// errchk $G $D/$F.go + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Issue 1722. + +// Check that the error messages says +// bug337.go:16: len("foo") not used +// and not +// bug337.go:16: 3 not used + +package main + +func main() { + len("foo") // ERROR "len" +} + diff --git a/test/fixedbugs/bug338.go b/test/fixedbugs/bug338.go new file mode 100644 index 000000000..c368a7fad --- /dev/null +++ b/test/fixedbugs/bug338.go @@ -0,0 +1,22 @@ +// $G $D/$F.go + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Issue 1787. + +package main + +import "unsafe" + +const x = unsafe.Sizeof([8]byte{}) + +func main() { + var b [x]int + _ = b +} + +/* +bug338.go:14: array bound must be non-negative +*/ diff --git a/test/fixedbugs/bug339.go b/test/fixedbugs/bug339.go new file mode 100644 index 000000000..eac7c5ee6 --- /dev/null +++ b/test/fixedbugs/bug339.go @@ -0,0 +1,20 @@ +// $G $D/$F.go && $L $F.$A && ./$A.out + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Issue 1608. +// Size used to be -1000000000. + +package main + +import "unsafe" + +func main() { + var a interface{} = 0 + size := unsafe.Sizeof(a) + if size != 2*unsafe.Sizeof((*int)(nil)) { + println("wrong size: ", size) + } +} diff --git a/test/fixedbugs/bug340.go b/test/fixedbugs/bug340.go new file mode 100644 index 000000000..461cc6cd4 --- /dev/null +++ b/test/fixedbugs/bug340.go @@ -0,0 +1,17 @@ +// errchk $G $D/$F.go + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Issue 1606. + +package main + +func main() { + var x interface{} + switch t := x.(type) { // ERROR "0 is not a type" + case 0: + t.x = 1 // ERROR "type interface \{ \}" + } +} diff --git a/test/fixedbugs/bug341.go b/test/fixedbugs/bug341.go new file mode 100644 index 000000000..8ee52e1ef --- /dev/null +++ b/test/fixedbugs/bug341.go @@ -0,0 +1,22 @@ +// $G $D/$F.go && $L $F.$A && ./$A.out || echo BUG: bug341 + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Used to panic because 8g was generating incorrect +// code for converting a negative float to a uint64. + +package main + +func main() { + var x float32 = -2.5 + + _ = uint64(x) + _ = float32(0) +} +/* +panic: runtime error: floating point error + +[signal 0x8 code=0x6 addr=0x8048c64 pc=0x8048c64] +*/ diff --git a/test/fixedbugs/bug342.go b/test/fixedbugs/bug342.go new file mode 100644 index 000000000..0852cdd34 --- /dev/null +++ b/test/fixedbugs/bug342.go @@ -0,0 +1,24 @@ +// errchk $G $D/$F.go + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Issue 1871. + +package p + +type a interface { + foo(x int) (x int) // ERROR "redeclared|redefinition" +} + +var b interface { + bar(y int) (y int) // ERROR "redeclared|redefinition" +} + +/* +Previously: + +bug.go:1 x redclared in this block + previous declaration at bug.go:1 +*/ diff --git a/test/fixedbugs/bug343.go b/test/fixedbugs/bug343.go new file mode 100644 index 000000000..efc87e3d7 --- /dev/null +++ b/test/fixedbugs/bug343.go @@ -0,0 +1,33 @@ +// $G $D/$F.go && $L $F.$A && ./$A.out || echo BUG: bug343 + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// issue 1900 + +package main + +func getArgs(data map[string]interface{}, keys ...string) map[string]string { + ret := map[string]string{} + var ok bool + for _, k := range keys { + ret[k], ok = data[k].(string) + if !ok {} + } + return ret +} + +func main() { + x := getArgs(map[string]interface{}{"x":"y"}, "x") + if x["x"] != "y" { + println("BUG bug343", x) + } +} + + +/* +typecheck [1008592b0] +. INDREG a(1) l(15) x(24) tc(2) runtime.ret G0 string +bug343.go:15: internal compiler error: typecheck INDREG +*/ diff --git a/test/golden.out b/test/golden.out index 725e8de44..4400e41dd 100644 --- a/test/golden.out +++ b/test/golden.out @@ -161,8 +161,3 @@ panic: interface conversion: interface is main.T, not main.T 0x0 == bugs/ - -=========== bugs/bug322.go -bugs/bug322.dir/main.go:19: implicit assignment of unexported field 'x' of lib.T in method receiver -bugs/bug322.dir/main.go:32: implicit assignment of unexported field 'x' of lib.T in method receiver -BUG: fails incorrectly diff --git a/test/shift1.go b/test/shift1.go new file mode 100644 index 000000000..8fa48a03c --- /dev/null +++ b/test/shift1.go @@ -0,0 +1,36 @@ +// errchk $G -e $D/$F.go + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Issue 1708, illegal cases. + +package p + +func f(x int) int { return 0 } +func g(x interface{}) int { return 0 } +func h(x float64) int { return 0 } + +// from the spec +var ( + s uint = 33 + u = 1.0 << s // ERROR "invalid operation" + v float32 = 1 << s // ERROR "invalid operation" +) + +// non-constant shift expressions +var ( + e1 = g(2.0 << s) // ERROR "invalid operation" + f1 = h(2 << s) // ERROR "invalid operation" + g1 int64 = 1.1 << s // ERROR "truncated" +) + +// constant shift expressions +const c uint = 65 + +var ( + a2 int = 1.0 << c // ERROR "overflow" + b2 = 1.0 << c // ERROR "overflow" + d2 = f(1.0 << c) // ERROR "overflow" +) diff --git a/test/shift2.go b/test/shift2.go new file mode 100644 index 000000000..ec4c7addc --- /dev/null +++ b/test/shift2.go @@ -0,0 +1,42 @@ +// $G $D/$F.go || echo BUG: shift2 + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Issue 1708, legal cases. + +package p + +func f(x int) int { return 0 } +func g(x interface{}) int { return 0 } +func h(x float64) int { return 0 } + +// from the spec +var ( + s uint = 33 + i = 1 << s // 1 has type int + j int32 = 1 << s // 1 has type int32; j == 0 + k = uint64(1 << s) // 1 has type uint64; k == 1<<33 + m int = 1.0 << s // legal: 1.0 has type int + w int64 = 1.0 << 33 // legal: 1.0<<33 is a constant shift expression +) + +// non-constant shift expressions +var ( + a1 int = 2.0 << s // typeof(2.0) is int in this context => legal shift + d1 = f(2.0 << s) // typeof(2.0) is int in this context => legal shift +) + +// constant shift expressions +const c uint = 5 + +var ( + a2 int = 2.0 << c // a2 == 64 (type int) + b2 = 2.0 << c // b2 == 64 (untyped integer) + _ = f(b2) // verify b2 has type int + c2 float64 = 2 << c // c2 == 64.0 (type float64) + d2 = f(2.0 << c) // == f(64) + e2 = g(2.0 << c) // == g(int(64)) + f2 = h(2 << c) // == h(float64(64.0)) +) |
