path: root/src/cmd/goapi/goapi.go

// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Goapi computes the exported API of a set of Go packages.
package main

import (
	"bufio"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/doc"
	"go/parser"
	"go/printer"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
)

// Flags
var (
	checkFile = flag.String("c", "", "optional filename to check API against")
	verbose   = flag.Bool("v", false, "Verbose debugging")
)

func main() {
	flag.Parse()

	var pkgs []string
	if flag.NArg() > 0 {
		pkgs = flag.Args()
	} else {
		stds, err := exec.Command("go", "list", "std").Output()
		if err != nil {
			log.Fatal(err)
		}
		pkgs = strings.Fields(string(stds))
	}

	w := NewWalker()
	tree, _, err := build.FindTree("os") // some known package
	if err != nil {
		log.Fatalf("failed to find tree: %v", err)
	}

	for _, pkg := range pkgs {
		if strings.HasPrefix(pkg, "cmd/") ||
			strings.HasPrefix(pkg, "exp/") ||
			strings.HasPrefix(pkg, "old/") {
			continue
		}
		if !tree.HasSrc(pkg) {
			log.Fatalf("no source in tree for package %q", pkg)
		}
		pkgSrcDir := filepath.Join(tree.SrcDir(), filepath.FromSlash(pkg))
		w.WalkPackage(pkg, pkgSrcDir)
	}

	bw := bufio.NewWriter(os.Stdout)
	defer bw.Flush()

	if *checkFile != "" {
		bs, err := ioutil.ReadFile(*checkFile)
		if err != nil {
			log.Fatalf("Error reading file %s: %v", *checkFile, err)
		}
		v1 := strings.Split(string(bs), "\n")
		sort.Strings(v1)
		v2 := w.Features()
		take := func(sl *[]string) string {
			s := (*sl)[0]
			*sl = (*sl)[1:]
			return s
		}
		for len(v1) > 0 || len(v2) > 0 {
			switch {
			case len(v2) == 0 || v1[0] < v2[0]:
				fmt.Fprintf(bw, "-%s\n", take(&v1))
			case len(v1) == 0 || v1[0] > v2[0]:
				fmt.Fprintf(bw, "+%s\n", take(&v2))
			default:
				take(&v1)
				take(&v2)
			}
		}
	} else {
		for _, f := range w.Features() {
			fmt.Fprintf(bw, "%s\n", f)
		}
	}
}

type Walker struct {
	fset           *token.FileSet
	scope          []string
	features       map[string]bool // set
	lastConstType  string
	curPackageName string
	curPackage     *ast.Package
	prevConstType  map[string]string // identifer -> "ideal-int"
}

func NewWalker() *Walker {
	return &Walker{
		fset:     token.NewFileSet(),
		features: make(map[string]bool),
	}
}

// hardCodedConstantType is a hack until the type checker is sufficient for our needs.
// Rather than litter the code with unnecessary type annotations, we'll hard-code
// the cases we can't handle yet.
func (w *Walker) hardCodedConstantType(name string) (typ string, ok bool) {
	switch w.scope[0] {
	case "pkg compress/gzip", "pkg compress/zlib":
		switch name {
		case "NoCompression", "BestSpeed", "BestCompression", "DefaultCompression":
			return "ideal-int", true
		}
	case "pkg os":
		switch name {
		case "WNOHANG", "WSTOPPED", "WUNTRACED":
			return "ideal-int", true
		}
	case "pkg path/filepath":
		switch name {
		case "Separator", "ListSeparator":
			return "char", true
		}
	case "pkg unicode/utf8":
		switch name {
		case "RuneError":
			return "char", true
		}
	case "pkg text/scanner":
		// TODO: currently this tool only resolves const types
		// that reference other constant types if they appear
		// in the right order.  the scanner package has
		// ScanIdents and such coming before the Ident/Int/etc
		// tokens, hence this hack.
		if strings.HasPrefix(name, "Scan") || name == "SkipComments" {
			return "ideal-int", true
		}
	}
	return "", false
}

func (w *Walker) Features() (fs []string) {
	for f := range w.features {
		fs = append(fs, f)
	}
	sort.Strings(fs)
	return
}

func (w *Walker) WalkPackage(name, dir string) {
	log.Printf("package %s", name)
	pop := w.pushScope("pkg " + name)
	defer pop()

	info, err := build.ScanDir(dir)
	if err != nil {
		log.Fatalf("pkg %q, dir %q: ScanDir: %v", name, dir, err)
	}

	apkg := &ast.Package{
		Files: make(map[string]*ast.File),
	}

	files := append(append([]string{}, info.GoFiles...), info.CgoFiles...)
	for _, file := range files {
		f, err := parser.ParseFile(w.fset, filepath.Join(dir, file), nil, 0)
		if err != nil {
			log.Fatalf("error parsing package %s, file %s: %v", name, file, err)
		}
		apkg.Files[file] = f
	}

	w.curPackageName = name
	w.curPackage = apkg
	w.prevConstType = map[string]string{}
	for name, afile := range apkg.Files {
		w.walkFile(filepath.Join(dir, name), afile)
	}

	// Now that we're done walking types, vars and consts
	// in the *ast.Package, use go/doc to do the rest
	// (functions and methods). This is done here because
	// go/doc is destructive.  We can't use the
	// *ast.Package after this.
	dpkg := doc.New(apkg, name, 0)

	for _, t := range dpkg.Types {
		// Move funcs up to the top-level, not hiding in the Types.
		dpkg.Funcs = append(dpkg.Funcs, t.Funcs...)

		for _, m := range t.Methods {
			w.walkFuncDecl(m.Decl)
		}
	}

	for _, f := range dpkg.Funcs {
		w.walkFuncDecl(f.Decl)
	}
}

// pushScope enters a new scope (walking a package, type, node, etc)
// and returns a function that will leave the scope (with sanity checking
// for mismatched pushes & pops)
func (w *Walker) pushScope(name string) (popFunc func()) {
	w.scope = append(w.scope, name)
	return func() {
		if len(w.scope) == 0 {
			log.Fatalf("attempt to leave scope %q with empty scope list", name)
		}
		if w.scope[len(w.scope)-1] != name {
			log.Fatalf("attempt to leave scope %q, but scope is currently %#v", name, w.scope)
		}
		w.scope = w.scope[:len(w.scope)-1]
	}
}

func (w *Walker) walkFile(name string, file *ast.File) {
	// Not entering a scope here; file boundaries aren't interesting.

	for _, di := range file.Decls {
		switch d := di.(type) {
		case *ast.GenDecl:
			switch d.Tok {
			case token.IMPORT:
				continue
			case token.CONST:
				for _, sp := range d.Specs {
					w.walkConst(sp.(*ast.ValueSpec))
				}
			case token.TYPE:
				for _, sp := range d.Specs {
					w.walkTypeSpec(sp.(*ast.TypeSpec))
				}
			case token.VAR:
				for _, sp := range d.Specs {
					w.walkVar(sp.(*ast.ValueSpec))
				}
			default:
				log.Fatalf("unknown token type %d in GenDecl", d.Tok)
			}
		case *ast.FuncDecl:
			// Ignore. Handled in subsequent pass, by go/doc.
		default:
			log.Printf("unhandled %T, %#v\n", di, di)
			printer.Fprint(os.Stderr, w.fset, di)
			os.Stderr.Write([]byte("\n"))
		}
	}
}

var constType = map[token.Token]string{
	token.INT:    "ideal-int",
	token.FLOAT:  "ideal-float",
	token.STRING: "ideal-string",
	token.CHAR:   "ideal-char",
	token.IMAG:   "ideal-imag",
}

var varType = map[token.Token]string{
	token.INT:    "int",
	token.FLOAT:  "float64",
	token.STRING: "string",
	token.CHAR:   "rune",
	token.IMAG:   "complex128",
}

var errTODO = errors.New("TODO")

func (w *Walker) constValueType(vi interface{}) (string, error) {
	switch v := vi.(type) {
	case *ast.BasicLit:
		litType, ok := constType[v.Kind]
		if !ok {
			return "", fmt.Errorf("unknown basic literal kind %#v", v)
		}
		return litType, nil
	case *ast.UnaryExpr:
		return w.constValueType(v.X)
	case *ast.SelectorExpr:
		// e.g. compress/gzip's BestSpeed == flate.BestSpeed
		return "", errTODO
	case *ast.Ident:
		if v.Name == "iota" {
			return "ideal-int", nil // hack.
		}
		if v.Name == "false" || v.Name == "true" {
			return "ideal-bool", nil
		}
		if v.Name == "intSize" && w.curPackageName == "strconv" {
			// Hack.
			return "ideal-int", nil
		}
		if t, ok := w.prevConstType[v.Name]; ok {
			return t, nil
		}
		return "", fmt.Errorf("can't resolve existing constant %q", v.Name)
	case *ast.BinaryExpr:
		left, err := w.constValueType(v.X)
		if err != nil {
			return "", err
		}
		right, err := w.constValueType(v.Y)
		if err != nil {
			return "", err
		}
		if left != right {
			if left == "ideal-int" && right == "ideal-float" {
				return "ideal-float", nil // math.Log2E
			}
			if left == "ideal-char" && right == "ideal-int" {
				return "ideal-int", nil // math/big.MaxBase
			}
			if left == "ideal-int" && right == "ideal-char" {
				return "ideal-int", nil // text/scanner.GoWhitespace
			}
			if left == "ideal-int" && right == "Duration" {
				// Hack, for package time.
				return "Duration", nil
			}
			return "", fmt.Errorf("in BinaryExpr, unhandled type mismatch; left=%q, right=%q", left, right)
		}
		return left, nil
	case *ast.CallExpr:
		// Not a call, but a type conversion.
		return w.nodeString(v.Fun), nil
	case *ast.ParenExpr:
		return w.constValueType(v.X)
	}
	return "", fmt.Errorf("unknown const value type %T", vi)
}

func (w *Walker) varValueType(vi interface{}) (string, error) {
	valStr := w.nodeString(vi)
	if strings.HasPrefix(valStr, "errors.New(") {
		return "error", nil
	}

	switch v := vi.(type) {
	case *ast.BasicLit:
		litType, ok := varType[v.Kind]
		if !ok {
			return "", fmt.Errorf("unknown basic literal kind %#v", v)
		}
		return litType, nil
	case *ast.CompositeLit:
		return w.nodeString(v.Type), nil
	case *ast.FuncLit:
		return w.nodeString(w.namelessType(v.Type)), nil
	case *ast.UnaryExpr:
		if v.Op == token.AND {
			typ, err := w.varValueType(v.X)
			return "*" + typ, err
		}
		return "", fmt.Errorf("unknown unary expr: %#v", v)
	case *ast.SelectorExpr:
		return "", errTODO
	case *ast.Ident:
		node, _, ok := w.resolveName(v.Name)
		if !ok {
			return "", fmt.Errorf("unresolved identifier: %q", v.Name)
		}
		return w.varValueType(node)
	case *ast.BinaryExpr:
		left, err := w.varValueType(v.X)
		if err != nil {
			return "", err
		}
		right, err := w.varValueType(v.Y)
		if err != nil {
			return "", err
		}
		if left != right {
			return "", fmt.Errorf("in BinaryExpr, unhandled type mismatch; left=%q, right=%q", left, right)
		}
		return left, nil
	case *ast.ParenExpr:
		return w.varValueType(v.X)
	case *ast.CallExpr:
		funStr := w.nodeString(v.Fun)
		node, _, ok := w.resolveName(funStr)
		if !ok {
			return "", fmt.Errorf("unresolved named %q", funStr)
		}
		if funcd, ok := node.(*ast.FuncDecl); ok {
			// Assume at the top level that all functions have exactly 1 result
			return w.nodeString(w.namelessType(funcd.Type.Results.List[0].Type)), nil
		}
		// maybe a function call; maybe a conversion.  Need to lookup type.
		return "", fmt.Errorf("resolved name %q to a %T: %#v", funStr, node, node)
	default:
		return "", fmt.Errorf("unknown const value type %T", vi)
	}
	panic("unreachable")
}

// resolveName finds a top-level node named name and returns the node
// v and its type t, if known.
func (w *Walker) resolveName(name string) (v interface{}, t interface{}, ok bool) {
	for _, file := range w.curPackage.Files {
		for _, di := range file.Decls {
			switch d := di.(type) {
			case *ast.FuncDecl:
				if d.Name.Name == name {
					return d, d.Type, true
				}
			case *ast.GenDecl:
				switch d.Tok {
				case token.TYPE:
					for _, sp := range d.Specs {
						ts := sp.(*ast.TypeSpec)
						if ts.Name.Name == name {
							return ts, ts.Type, true
						}
					}
				case token.VAR:
					for _, sp := range d.Specs {
						vs := sp.(*ast.ValueSpec)
						for i, vname := range vs.Names {
							if vname.Name == name {
								if len(vs.Values) > i {
									return vs.Values[i], vs.Type, true
								}
								return nil, vs.Type, true
							}
						}
					}
				}
			}
		}
	}
	return nil, nil, false
}

func (w *Walker) walkConst(vs *ast.ValueSpec) {
	for _, ident := range vs.Names {
		if !ast.IsExported(ident.Name) {
			continue
		}
		litType := ""
		if vs.Type != nil {
			litType = w.nodeString(vs.Type)
		} else {
			litType = w.lastConstType
			if vs.Values != nil {
				if len(vs.Values) != 1 {
					log.Fatalf("const %q, values: %#v", ident.Name, vs.Values)
				}
				var err error
				litType, err = w.constValueType(vs.Values[0])
				if err != nil {
					if t, ok := w.hardCodedConstantType(ident.Name); ok {
						litType = t
						err = nil
					} else {
						log.Fatalf("unknown kind in const %q (%T): %v", ident.Name, vs.Values[0], err)
					}
				}
			}
		}
		if litType == "" {
			log.Fatalf("unknown kind in const %q", ident.Name)
		}
		w.lastConstType = litType

		w.emitFeature(fmt.Sprintf("const %s %s", ident, litType))
		w.prevConstType[ident.Name] = litType
	}
}

func (w *Walker) walkVar(vs *ast.ValueSpec) {
	for i, ident := range vs.Names {
		if !ast.IsExported(ident.Name) {
			continue
		}

		typ := ""
		if vs.Type != nil {
			typ = w.nodeString(vs.Type)
		} else {
			if len(vs.Values) == 0 {
				log.Fatalf("no values for var %q", ident.Name)
			}
			if len(vs.Values) > 1 {
				log.Fatalf("more than 1 values in ValueSpec not handled, var %q", ident.Name)
			}
			var err error
			typ, err = w.varValueType(vs.Values[i])
			if err != nil {
				log.Fatalf("unknown type of variable %q, type %T, error = %v\ncode: %s",
					ident.Name, vs.Values[i], err, w.nodeString(vs.Values[i]))
			}
		}
		w.emitFeature(fmt.Sprintf("var %s %s", ident, typ))
	}
}

func (w *Walker) nodeString(node interface{}) string {
	if node == nil {
		return ""
	}
	var b bytes.Buffer
	printer.Fprint(&b, w.fset, node)
	return b.String()
}

func (w *Walker) nodeDebug(node interface{}) string {
	if node == nil {
		return ""
	}
	var b bytes.Buffer
	ast.Fprint(&b, w.fset, node, nil)
	return b.String()
}

func (w *Walker) walkTypeSpec(ts *ast.TypeSpec) {
	name := ts.Name.Name
	if !ast.IsExported(name) {
		return
	}

	switch t := ts.Type.(type) {
	case *ast.StructType:
		w.walkStructType(name, t)
	case *ast.InterfaceType:
		w.walkInterfaceType(name, t)
	default:
		w.emitFeature(fmt.Sprintf("type %s %s", name, w.nodeString(ts.Type)))
		//log.Fatalf("unknown typespec %T", ts.Type)
	}
}

func (w *Walker) walkStructType(name string, t *ast.StructType) {
	typeStruct := fmt.Sprintf("type %s struct", name)
	w.emitFeature(typeStruct)
	pop := w.pushScope(typeStruct)
	defer pop()
	for _, f := range t.Fields.List {
		typ := f.Type
		for _, name := range f.Names {
			if ast.IsExported(name.Name) {
				w.emitFeature(fmt.Sprintf("%s %s", name, w.nodeString(w.namelessType(typ))))
			}
		}
		if f.Names == nil {
			switch v := typ.(type) {
			case *ast.Ident:
				if ast.IsExported(v.Name) {
					w.emitFeature(fmt.Sprintf("embedded %s", v.Name))
				}
			case *ast.StarExpr:
				switch vv := v.X.(type) {
				case *ast.Ident:
					if ast.IsExported(vv.Name) {
						w.emitFeature(fmt.Sprintf("embedded *%s", vv.Name))
					}
				case *ast.SelectorExpr:
					w.emitFeature(fmt.Sprintf("embedded %s", w.nodeString(typ)))
				default:
					log.Fatal("unable to handle embedded starexpr before %T", typ)
				}
			case *ast.SelectorExpr:
				w.emitFeature(fmt.Sprintf("embedded %s", w.nodeString(typ)))
			default:
				log.Fatalf("unable to handle embedded %T", typ)
			}
		}
	}
}

func (w *Walker) walkInterfaceType(name string, t *ast.InterfaceType) {
	methods := []string{}

	pop := w.pushScope("type " + name + " interface")
	for _, f := range t.Methods.List {
		typ := f.Type
		for _, name := range f.Names {
			if ast.IsExported(name.Name) {
				ft := typ.(*ast.FuncType)
				w.emitFeature(fmt.Sprintf("%s%s", name, w.funcSigString(ft)))
				methods = append(methods, name.Name)
			}
		}
	}
	pop()

	sort.Strings(methods)
	if len(methods) == 0 {
		w.emitFeature(fmt.Sprintf("type %s interface {}", name))
	} else {
		w.emitFeature(fmt.Sprintf("type %s interface { %s }", name, strings.Join(methods, ", ")))
	}
}

func (w *Walker) walkFuncDecl(f *ast.FuncDecl) {
	if !ast.IsExported(f.Name.Name) {
		return
	}
	if f.Recv != nil {
		// Method.
		recvType := w.nodeString(f.Recv.List[0].Type)
		keep := ast.IsExported(recvType) ||
			(strings.HasPrefix(recvType, "*") &&
				ast.IsExported(recvType[1:]))
		if !keep {
			return
		}
		w.emitFeature(fmt.Sprintf("method (%s) %s%s", recvType, f.Name.Name, w.funcSigString(f.Type)))
		return
	}
	// Else, a function
	w.emitFeature(fmt.Sprintf("func %s%s", f.Name.Name, w.funcSigString(f.Type)))
}

func (w *Walker) funcSigString(ft *ast.FuncType) string {
	var b bytes.Buffer
	b.WriteByte('(')
	if ft.Params != nil {
		for i, f := range ft.Params.List {
			if i > 0 {
				b.WriteString(", ")
			}
			b.WriteString(w.nodeString(w.namelessType(f.Type)))
		}
	}
	b.WriteByte(')')
	if ft.Results != nil {
		if nr := len(ft.Results.List); nr > 0 {
			b.WriteByte(' ')
			if nr > 1 {
				b.WriteByte('(')
			}
			for i, f := range ft.Results.List {
				if i > 0 {
					b.WriteString(", ")
				}
				b.WriteString(w.nodeString(w.namelessType(f.Type)))
			}
			if nr > 1 {
				b.WriteByte(')')
			}
		}
	}
	return b.String()
}

// namelessType returns a type node that lacks any variable names.
func (w *Walker) namelessType(t interface{}) interface{} {
	ft, ok := t.(*ast.FuncType)
	if !ok {
		return t
	}
	return &ast.FuncType{
		Params:  w.namelessFieldList(ft.Params),
		Results: w.namelessFieldList(ft.Results),
	}
}

// namelessFieldList returns a deep clone of fl, with the cloned fields
// lacking names.
func (w *Walker) namelessFieldList(fl *ast.FieldList) *ast.FieldList {
	fl2 := &ast.FieldList{}
	if fl != nil {
		for _, f := range fl.List {
			fl2.List = append(fl2.List, w.namelessField(f))
		}
	}
	return fl2
}

// namelessField clones f, but not preserving the names of fields.
// (comments and tags are also ignored)
func (w *Walker) namelessField(f *ast.Field) *ast.Field {
	return &ast.Field{
		Type: f.Type,
	}
}

func (w *Walker) emitFeature(feature string) {
	f := strings.Join(w.scope, ", ") + ", " + feature
	if _, dup := w.features[f]; dup {
		panic("duplicate feature inserted: " + f)
	}

	if strings.Contains(f, "\n") {
		// TODO: for now, just skip over the
		// runtime.MemStatsType.BySize type, which this tool
		// doesn't properly handle. It's pretty low-level,
		// though, so not super important to protect against.
		if strings.HasPrefix(f, "pkg runtime") && strings.Contains(f, "BySize [61]struct") {
			return
		}
		panic("feature contains newlines: " + f)
	}
	w.features[f] = true
	if *verbose {
		log.Printf("feature: %s", f)
	}
}

func strListContains(l []string, s string) bool {
	for _, v := range l {
		if v == s {
			return true
		}
	}
	return false
}