The Go Programming Language

Source file src/pkg/go/doc/doc.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// Package doc extracts source code documentation from a Go AST.
     6	package doc
     7	
     8	import (
     9		"go/ast"
    10		"go/token"
    11		"regexp"
    12		"sort"
    13	)
    14	
    15	// ----------------------------------------------------------------------------
    16	
    17	type typeDoc struct {
    18		// len(decl.Specs) == 1, and the element type is *ast.TypeSpec
    19		// if the type declaration hasn't been seen yet, decl is nil
    20		decl *ast.GenDecl
    21		// values, factory functions, and methods associated with the type
    22		values    []*ast.GenDecl // consts and vars
    23		factories map[string]*ast.FuncDecl
    24		methods   map[string]*ast.FuncDecl
    25	}
    26	
    27	// docReader accumulates documentation for a single package.
    28	// It modifies the AST: Comments (declaration documentation)
    29	// that have been collected by the DocReader are set to nil
    30	// in the respective AST nodes so that they are not printed
    31	// twice (once when printing the documentation and once when
    32	// printing the corresponding AST node).
    33	//
    34	type docReader struct {
    35		doc     *ast.CommentGroup // package documentation, if any
    36		pkgName string
    37		values  []*ast.GenDecl // consts and vars
    38		types   map[string]*typeDoc
    39		funcs   map[string]*ast.FuncDecl
    40		bugs    []*ast.CommentGroup
    41	}
    42	
    43	func (doc *docReader) init(pkgName string) {
    44		doc.pkgName = pkgName
    45		doc.types = make(map[string]*typeDoc)
    46		doc.funcs = make(map[string]*ast.FuncDecl)
    47	}
    48	
    49	func (doc *docReader) addDoc(comments *ast.CommentGroup) {
    50		if doc.doc == nil {
    51			// common case: just one package comment
    52			doc.doc = comments
    53			return
    54		}
    55	
    56		// More than one package comment: Usually there will be only
    57		// one file with a package comment, but it's better to collect
    58		// all comments than drop them on the floor.
    59		// (This code isn't particularly clever - no amortized doubling is
    60		// used - but this situation occurs rarely and is not time-critical.)
    61		n1 := len(doc.doc.List)
    62		n2 := len(comments.List)
    63		list := make([]*ast.Comment, n1+1+n2) // + 1 for separator line
    64		copy(list, doc.doc.List)
    65		list[n1] = &ast.Comment{token.NoPos, "//"} // separator line
    66		copy(list[n1+1:], comments.List)
    67		doc.doc = &ast.CommentGroup{list}
    68	}
    69	
    70	func (doc *docReader) addType(decl *ast.GenDecl) {
    71		spec := decl.Specs[0].(*ast.TypeSpec)
    72		typ := doc.lookupTypeDoc(spec.Name.Name)
    73		// typ should always be != nil since declared types
    74		// are always named - be conservative and check
    75		if typ != nil {
    76			// a type should be added at most once, so typ.decl
    77			// should be nil - if it isn't, simply overwrite it
    78			typ.decl = decl
    79		}
    80	}
    81	
    82	func (doc *docReader) lookupTypeDoc(name string) *typeDoc {
    83		if name == "" {
    84			return nil // no type docs for anonymous types
    85		}
    86		if tdoc, found := doc.types[name]; found {
    87			return tdoc
    88		}
    89		// type wasn't found - add one without declaration
    90		tdoc := &typeDoc{nil, nil, make(map[string]*ast.FuncDecl), make(map[string]*ast.FuncDecl)}
    91		doc.types[name] = tdoc
    92		return tdoc
    93	}
    94	
    95	func baseTypeName(typ ast.Expr) string {
    96		switch t := typ.(type) {
    97		case *ast.Ident:
    98			// if the type is not exported, the effect to
    99			// a client is as if there were no type name
   100			if t.IsExported() {
   101				return t.Name
   102			}
   103		case *ast.StarExpr:
   104			return baseTypeName(t.X)
   105		}
   106		return ""
   107	}
   108	
   109	func (doc *docReader) addValue(decl *ast.GenDecl) {
   110		// determine if decl should be associated with a type
   111		// Heuristic: For each typed entry, determine the type name, if any.
   112		//            If there is exactly one type name that is sufficiently
   113		//            frequent, associate the decl with the respective type.
   114		domName := ""
   115		domFreq := 0
   116		prev := ""
   117		for _, s := range decl.Specs {
   118			if v, ok := s.(*ast.ValueSpec); ok {
   119				name := ""
   120				switch {
   121				case v.Type != nil:
   122					// a type is present; determine its name
   123					name = baseTypeName(v.Type)
   124				case decl.Tok == token.CONST:
   125					// no type is present but we have a constant declaration;
   126					// use the previous type name (w/o more type information
   127					// we cannot handle the case of unnamed variables with
   128					// initializer expressions except for some trivial cases)
   129					name = prev
   130				}
   131				if name != "" {
   132					// entry has a named type
   133					if domName != "" && domName != name {
   134						// more than one type name - do not associate
   135						// with any type
   136						domName = ""
   137						break
   138					}
   139					domName = name
   140					domFreq++
   141				}
   142				prev = name
   143			}
   144		}
   145	
   146		// determine values list
   147		const threshold = 0.75
   148		values := &doc.values
   149		if domName != "" && domFreq >= int(float64(len(decl.Specs))*threshold) {
   150			// typed entries are sufficiently frequent
   151			typ := doc.lookupTypeDoc(domName)
   152			if typ != nil {
   153				values = &typ.values // associate with that type
   154			}
   155		}
   156	
   157		*values = append(*values, decl)
   158	}
   159	
   160	// Helper function to set the table entry for function f. Makes sure that
   161	// at least one f with associated documentation is stored in table, if there
   162	// are multiple f's with the same name.
   163	func setFunc(table map[string]*ast.FuncDecl, f *ast.FuncDecl) {
   164		name := f.Name.Name
   165		if g, exists := table[name]; exists && g.Doc != nil {
   166			// a function with the same name has already been registered;
   167			// since it has documentation, assume f is simply another
   168			// implementation and ignore it
   169			// TODO(gri) consider collecting all functions, or at least
   170			//           all comments
   171			return
   172		}
   173		// function doesn't exist or has no documentation; use f
   174		table[name] = f
   175	}
   176	
   177	func (doc *docReader) addFunc(fun *ast.FuncDecl) {
   178		name := fun.Name.Name
   179	
   180		// determine if it should be associated with a type
   181		if fun.Recv != nil {
   182			// method
   183			typ := doc.lookupTypeDoc(baseTypeName(fun.Recv.List[0].Type))
   184			if typ != nil {
   185				// exported receiver type
   186				setFunc(typ.methods, fun)
   187			}
   188			// otherwise don't show the method
   189			// TODO(gri): There may be exported methods of non-exported types
   190			// that can be called because of exported values (consts, vars, or
   191			// function results) of that type. Could determine if that is the
   192			// case and then show those methods in an appropriate section.
   193			return
   194		}
   195	
   196		// perhaps a factory function
   197		// determine result type, if any
   198		if fun.Type.Results.NumFields() >= 1 {
   199			res := fun.Type.Results.List[0]
   200			if len(res.Names) <= 1 {
   201				// exactly one (named or anonymous) result associated
   202				// with the first type in result signature (there may
   203				// be more than one result)
   204				tname := baseTypeName(res.Type)
   205				typ := doc.lookupTypeDoc(tname)
   206				if typ != nil {
   207					// named and exported result type
   208	
   209					// Work-around for failure of heuristic: In package os
   210					// too many functions are considered factory functions
   211					// for the Error type. Eliminate manually for now as
   212					// this appears to be the only important case in the
   213					// current library where the heuristic fails.
   214					if doc.pkgName == "os" && tname == "Error" &&
   215						name != "NewError" && name != "NewSyscallError" {
   216						// not a factory function for os.Error
   217						setFunc(doc.funcs, fun) // treat as ordinary function
   218						return
   219					}
   220	
   221					setFunc(typ.factories, fun)
   222					return
   223				}
   224			}
   225		}
   226	
   227		// ordinary function
   228		setFunc(doc.funcs, fun)
   229	}
   230	
   231	func (doc *docReader) addDecl(decl ast.Decl) {
   232		switch d := decl.(type) {
   233		case *ast.GenDecl:
   234			if len(d.Specs) > 0 {
   235				switch d.Tok {
   236				case token.CONST, token.VAR:
   237					// constants and variables are always handled as a group
   238					doc.addValue(d)
   239				case token.TYPE:
   240					// types are handled individually
   241					for _, spec := range d.Specs {
   242						// make a (fake) GenDecl node for this TypeSpec
   243						// (we need to do this here - as opposed to just
   244						// for printing - so we don't lose the GenDecl
   245						// documentation)
   246						//
   247						// TODO(gri): Consider just collecting the TypeSpec
   248						// node (and copy in the GenDecl.doc if there is no
   249						// doc in the TypeSpec - this is currently done in
   250						// makeTypeDocs below). Simpler data structures, but
   251						// would lose GenDecl documentation if the TypeSpec
   252						// has documentation as well.
   253						doc.addType(&ast.GenDecl{d.Doc, d.Pos(), token.TYPE, token.NoPos, []ast.Spec{spec}, token.NoPos})
   254						// A new GenDecl node is created, no need to nil out d.Doc.
   255					}
   256				}
   257			}
   258		case *ast.FuncDecl:
   259			doc.addFunc(d)
   260		}
   261	}
   262	
   263	func copyCommentList(list []*ast.Comment) []*ast.Comment {
   264		return append([]*ast.Comment(nil), list...)
   265	}
   266	
   267	var (
   268		bug_markers = regexp.MustCompile("^/[/*][ \t]*BUG\\(.*\\):[ \t]*") // BUG(uid):
   269		bug_content = regexp.MustCompile("[^ \n\r\t]+")                    // at least one non-whitespace char
   270	)
   271	
   272	// addFile adds the AST for a source file to the docReader.
   273	// Adding the same AST multiple times is a no-op.
   274	//
   275	func (doc *docReader) addFile(src *ast.File) {
   276		// add package documentation
   277		if src.Doc != nil {
   278			doc.addDoc(src.Doc)
   279			src.Doc = nil // doc consumed - remove from ast.File node
   280		}
   281	
   282		// add all declarations
   283		for _, decl := range src.Decls {
   284			doc.addDecl(decl)
   285		}
   286	
   287		// collect BUG(...) comments
   288		for _, c := range src.Comments {
   289			text := c.List[0].Text
   290			if m := bug_markers.FindStringIndex(text); m != nil {
   291				// found a BUG comment; maybe empty
   292				if btxt := text[m[1]:]; bug_content.MatchString(btxt) {
   293					// non-empty BUG comment; collect comment without BUG prefix
   294					list := copyCommentList(c.List)
   295					list[0].Text = text[m[1]:]
   296					doc.bugs = append(doc.bugs, &ast.CommentGroup{list})
   297				}
   298			}
   299		}
   300		src.Comments = nil // consumed unassociated comments - remove from ast.File node
   301	}
   302	
   303	func NewFileDoc(file *ast.File) *PackageDoc {
   304		var r docReader
   305		r.init(file.Name.Name)
   306		r.addFile(file)
   307		return r.newDoc("", nil)
   308	}
   309	
   310	func NewPackageDoc(pkg *ast.Package, importpath string) *PackageDoc {
   311		var r docReader
   312		r.init(pkg.Name)
   313		filenames := make([]string, len(pkg.Files))
   314		i := 0
   315		for filename, f := range pkg.Files {
   316			r.addFile(f)
   317			filenames[i] = filename
   318			i++
   319		}
   320		return r.newDoc(importpath, filenames)
   321	}
   322	
   323	// ----------------------------------------------------------------------------
   324	// Conversion to external representation
   325	
   326	// ValueDoc is the documentation for a group of declared
   327	// values, either vars or consts.
   328	//
   329	type ValueDoc struct {
   330		Doc   string
   331		Decl  *ast.GenDecl
   332		order int
   333	}
   334	
   335	type sortValueDoc []*ValueDoc
   336	
   337	func (p sortValueDoc) Len() int      { return len(p) }
   338	func (p sortValueDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
   339	
   340	func declName(d *ast.GenDecl) string {
   341		if len(d.Specs) != 1 {
   342			return ""
   343		}
   344	
   345		switch v := d.Specs[0].(type) {
   346		case *ast.ValueSpec:
   347			return v.Names[0].Name
   348		case *ast.TypeSpec:
   349			return v.Name.Name
   350		}
   351	
   352		return ""
   353	}
   354	
   355	func (p sortValueDoc) Less(i, j int) bool {
   356		// sort by name
   357		// pull blocks (name = "") up to top
   358		// in original order
   359		if ni, nj := declName(p[i].Decl), declName(p[j].Decl); ni != nj {
   360			return ni < nj
   361		}
   362		return p[i].order < p[j].order
   363	}
   364	
   365	func makeValueDocs(list []*ast.GenDecl, tok token.Token) []*ValueDoc {
   366		d := make([]*ValueDoc, len(list)) // big enough in any case
   367		n := 0
   368		for i, decl := range list {
   369			if decl.Tok == tok {
   370				d[n] = &ValueDoc{CommentText(decl.Doc), decl, i}
   371				n++
   372				decl.Doc = nil // doc consumed - removed from AST
   373			}
   374		}
   375		d = d[0:n]
   376		sort.Sort(sortValueDoc(d))
   377		return d
   378	}
   379	
   380	// FuncDoc is the documentation for a func declaration,
   381	// either a top-level function or a method function.
   382	//
   383	type FuncDoc struct {
   384		Doc  string
   385		Recv ast.Expr // TODO(rsc): Would like string here
   386		Name string
   387		Decl *ast.FuncDecl
   388	}
   389	
   390	type sortFuncDoc []*FuncDoc
   391	
   392	func (p sortFuncDoc) Len() int           { return len(p) }
   393	func (p sortFuncDoc) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
   394	func (p sortFuncDoc) Less(i, j int) bool { return p[i].Name < p[j].Name }
   395	
   396	func makeFuncDocs(m map[string]*ast.FuncDecl) []*FuncDoc {
   397		d := make([]*FuncDoc, len(m))
   398		i := 0
   399		for _, f := range m {
   400			doc := new(FuncDoc)
   401			doc.Doc = CommentText(f.Doc)
   402			f.Doc = nil // doc consumed - remove from ast.FuncDecl node
   403			if f.Recv != nil {
   404				doc.Recv = f.Recv.List[0].Type
   405			}
   406			doc.Name = f.Name.Name
   407			doc.Decl = f
   408			d[i] = doc
   409			i++
   410		}
   411		sort.Sort(sortFuncDoc(d))
   412		return d
   413	}
   414	
   415	// TypeDoc is the documentation for a declared type.
   416	// Consts and Vars are sorted lists of constants and variables of (mostly) that type.
   417	// Factories is a sorted list of factory functions that return that type.
   418	// Methods is a sorted list of method functions on that type.
   419	type TypeDoc struct {
   420		Doc       string
   421		Type      *ast.TypeSpec
   422		Consts    []*ValueDoc
   423		Vars      []*ValueDoc
   424		Factories []*FuncDoc
   425		Methods   []*FuncDoc
   426		Decl      *ast.GenDecl
   427		order     int
   428	}
   429	
   430	type sortTypeDoc []*TypeDoc
   431	
   432	func (p sortTypeDoc) Len() int      { return len(p) }
   433	func (p sortTypeDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
   434	func (p sortTypeDoc) Less(i, j int) bool {
   435		// sort by name
   436		// pull blocks (name = "") up to top
   437		// in original order
   438		if ni, nj := p[i].Type.Name.Name, p[j].Type.Name.Name; ni != nj {
   439			return ni < nj
   440		}
   441		return p[i].order < p[j].order
   442	}
   443	
   444	// NOTE(rsc): This would appear not to be correct for type ( )
   445	// blocks, but the doc extractor above has split them into
   446	// individual declarations.
   447	func (doc *docReader) makeTypeDocs(m map[string]*typeDoc) []*TypeDoc {
   448		d := make([]*TypeDoc, len(m))
   449		i := 0
   450		for _, old := range m {
   451			// all typeDocs should have a declaration associated with
   452			// them after processing an entire package - be conservative
   453			// and check
   454			if decl := old.decl; decl != nil {
   455				typespec := decl.Specs[0].(*ast.TypeSpec)
   456				t := new(TypeDoc)
   457				doc := typespec.Doc
   458				typespec.Doc = nil // doc consumed - remove from ast.TypeSpec node
   459				if doc == nil {
   460					// no doc associated with the spec, use the declaration doc, if any
   461					doc = decl.Doc
   462				}
   463				decl.Doc = nil // doc consumed - remove from ast.Decl node
   464				t.Doc = CommentText(doc)
   465				t.Type = typespec
   466				t.Consts = makeValueDocs(old.values, token.CONST)
   467				t.Vars = makeValueDocs(old.values, token.VAR)
   468				t.Factories = makeFuncDocs(old.factories)
   469				t.Methods = makeFuncDocs(old.methods)
   470				t.Decl = old.decl
   471				t.order = i
   472				d[i] = t
   473				i++
   474			} else {
   475				// no corresponding type declaration found - move any associated
   476				// values, factory functions, and methods back to the top-level
   477				// so that they are not lost (this should only happen if a package
   478				// file containing the explicit type declaration is missing or if
   479				// an unqualified type name was used after a "." import)
   480				// 1) move values
   481				doc.values = append(doc.values, old.values...)
   482				// 2) move factory functions
   483				for name, f := range old.factories {
   484					doc.funcs[name] = f
   485				}
   486				// 3) move methods
   487				for name, f := range old.methods {
   488					// don't overwrite functions with the same name
   489					if _, found := doc.funcs[name]; !found {
   490						doc.funcs[name] = f
   491					}
   492				}
   493			}
   494		}
   495		d = d[0:i] // some types may have been ignored
   496		sort.Sort(sortTypeDoc(d))
   497		return d
   498	}
   499	
   500	func makeBugDocs(list []*ast.CommentGroup) []string {
   501		d := make([]string, len(list))
   502		for i, g := range list {
   503			d[i] = CommentText(g)
   504		}
   505		return d
   506	}
   507	
   508	// PackageDoc is the documentation for an entire package.
   509	//
   510	type PackageDoc struct {
   511		PackageName string
   512		ImportPath  string
   513		Filenames   []string
   514		Doc         string
   515		Consts      []*ValueDoc
   516		Types       []*TypeDoc
   517		Vars        []*ValueDoc
   518		Funcs       []*FuncDoc
   519		Bugs        []string
   520	}
   521	
   522	// newDoc returns the accumulated documentation for the package.
   523	//
   524	func (doc *docReader) newDoc(importpath string, filenames []string) *PackageDoc {
   525		p := new(PackageDoc)
   526		p.PackageName = doc.pkgName
   527		p.ImportPath = importpath
   528		sort.Strings(filenames)
   529		p.Filenames = filenames
   530		p.Doc = CommentText(doc.doc)
   531		// makeTypeDocs may extend the list of doc.values and
   532		// doc.funcs and thus must be called before any other
   533		// function consuming those lists
   534		p.Types = doc.makeTypeDocs(doc.types)
   535		p.Consts = makeValueDocs(doc.values, token.CONST)
   536		p.Vars = makeValueDocs(doc.values, token.VAR)
   537		p.Funcs = makeFuncDocs(doc.funcs)
   538		p.Bugs = makeBugDocs(doc.bugs)
   539		return p
   540	}
   541	
   542	// ----------------------------------------------------------------------------
   543	// Filtering by name
   544	
   545	type Filter func(string) bool
   546	
   547	func matchFields(fields *ast.FieldList, f Filter) bool {
   548		if fields != nil {
   549			for _, field := range fields.List {
   550				for _, name := range field.Names {
   551					if f(name.Name) {
   552						return true
   553					}
   554				}
   555			}
   556		}
   557		return false
   558	}
   559	
   560	func matchDecl(d *ast.GenDecl, f Filter) bool {
   561		for _, d := range d.Specs {
   562			switch v := d.(type) {
   563			case *ast.ValueSpec:
   564				for _, name := range v.Names {
   565					if f(name.Name) {
   566						return true
   567					}
   568				}
   569			case *ast.TypeSpec:
   570				if f(v.Name.Name) {
   571					return true
   572				}
   573				switch t := v.Type.(type) {
   574				case *ast.StructType:
   575					if matchFields(t.Fields, f) {
   576						return true
   577					}
   578				case *ast.InterfaceType:
   579					if matchFields(t.Methods, f) {
   580						return true
   581					}
   582				}
   583			}
   584		}
   585		return false
   586	}
   587	
   588	func filterValueDocs(a []*ValueDoc, f Filter) []*ValueDoc {
   589		w := 0
   590		for _, vd := range a {
   591			if matchDecl(vd.Decl, f) {
   592				a[w] = vd
   593				w++
   594			}
   595		}
   596		return a[0:w]
   597	}
   598	
   599	func filterFuncDocs(a []*FuncDoc, f Filter) []*FuncDoc {
   600		w := 0
   601		for _, fd := range a {
   602			if f(fd.Name) {
   603				a[w] = fd
   604				w++
   605			}
   606		}
   607		return a[0:w]
   608	}
   609	
   610	func filterTypeDocs(a []*TypeDoc, f Filter) []*TypeDoc {
   611		w := 0
   612		for _, td := range a {
   613			n := 0 // number of matches
   614			if matchDecl(td.Decl, f) {
   615				n = 1
   616			} else {
   617				// type name doesn't match, but we may have matching consts, vars, factories or methods
   618				td.Consts = filterValueDocs(td.Consts, f)
   619				td.Vars = filterValueDocs(td.Vars, f)
   620				td.Factories = filterFuncDocs(td.Factories, f)
   621				td.Methods = filterFuncDocs(td.Methods, f)
   622				n += len(td.Consts) + len(td.Vars) + len(td.Factories) + len(td.Methods)
   623			}
   624			if n > 0 {
   625				a[w] = td
   626				w++
   627			}
   628		}
   629		return a[0:w]
   630	}
   631	
   632	// Filter eliminates documentation for names that don't pass through the filter f.
   633	// TODO: Recognize "Type.Method" as a name.
   634	//
   635	func (p *PackageDoc) Filter(f Filter) {
   636		p.Consts = filterValueDocs(p.Consts, f)
   637		p.Vars = filterValueDocs(p.Vars, f)
   638		p.Types = filterTypeDocs(p.Types, f)
   639		p.Funcs = filterFuncDocs(p.Funcs, f)
   640		p.Doc = "" // don't show top-level package doc
   641	}

release.r60.3. Except as noted, this content is licensed under a Creative Commons Attribution 3.0 License.