src/go/build/deps_test.go

   1 // Copyright 2012 The Go Authors.  All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // This file exercises the import parser but also checks that
   6 // some low-level packages do not have new dependencies added.
   7
   8 package build
   9
  10 import (
  11         "bytes"
  12         "fmt"
  13         "io/ioutil"
  14         "os"
  15         "path/filepath"
  16         "runtime"
  17         "sort"
  18         "strconv"
  19         "strings"
  20         "testing"
  21 )
  22
  23 // pkgDeps defines the expected dependencies between packages in
  24 // the Go source tree.  It is a statement of policy.
  25 // Changes should not be made to this map without prior discussion.
  26 //
  27 // The map contains two kinds of entries:
  28 // 1) Lower-case keys are standard import paths and list the
  29 // allowed imports in that package.
  30 // 2) Upper-case keys define aliases for package sets, which can then
  31 // be used as dependencies by other rules.
  32 //
  33 // DO NOT CHANGE THIS DATA TO FIX BUILDS.
  34 //
  35 var pkgDeps = map[string][]string{
  36         // L0 is the lowest level, core, nearly unavoidable packages.
  37         "errors":                  {},
  38         "io":                      {"errors", "sync"},
  39         "runtime":                 {"unsafe", "runtime/internal/atomic", "runtime/internal/sys"},
  40         "runtime/internal/sys":    {},
  41         "runtime/internal/atomic": {"unsafe"},
  42         "sync":        {"runtime", "sync/atomic", "unsafe"},
  43         "sync/atomic": {"unsafe"},
  44         "unsafe":      {},
  45
  46         "L0": {
  47                 "errors",
  48                 "io",
  49                 "runtime",
  50                 "runtime/internal/atomic",
  51                 "sync",
  52                 "sync/atomic",
  53                 "unsafe",
  54         },
  55
  56         // L1 adds simple functions and strings processing,
  57         // but not Unicode tables.
  58         "math":          {"unsafe"},
  59         "math/cmplx":    {"math"},
  60         "math/rand":     {"L0", "math"},
  61         "sort":          {},
  62         "strconv":       {"L0", "unicode/utf8", "math"},
  63         "unicode/utf16": {},
  64         "unicode/utf8":  {},
  65
  66         "L1": {
  67                 "L0",
  68                 "math",
  69                 "math/cmplx",
  70                 "math/rand",
  71                 "sort",
  72                 "strconv",
  73                 "unicode/utf16",
  74                 "unicode/utf8",
  75         },
  76
  77         // L2 adds Unicode and strings processing.
  78         "bufio":   {"L0", "unicode/utf8", "bytes"},
  79         "bytes":   {"L0", "unicode", "unicode/utf8"},
  80         "path":    {"L0", "unicode/utf8", "strings"},
  81         "strings": {"L0", "unicode", "unicode/utf8"},
  82         "unicode": {},
  83
  84         "L2": {
  85                 "L1",
  86                 "bufio",
  87                 "bytes",
  88                 "path",
  89                 "strings",
  90                 "unicode",
  91         },
  92
  93         // L3 adds reflection and some basic utility packages
  94         // and interface definitions, but nothing that makes
  95         // system calls.
  96         "crypto":              {"L2", "hash"},          // interfaces
  97         "crypto/cipher":       {"L2", "crypto/subtle"}, // interfaces
  98         "crypto/subtle":       {},
  99         "encoding/base32":     {"L2"},
 100         "encoding/base64":     {"L2"},
 101         "encoding/binary":     {"L2", "reflect"},
 102         "hash":                {"L2"}, // interfaces
 103         "hash/adler32":        {"L2", "hash"},
 104         "hash/crc32":          {"L2", "hash"},
 105         "hash/crc64":          {"L2", "hash"},
 106         "hash/fnv":            {"L2", "hash"},
 107         "image":               {"L2", "image/color"}, // interfaces
 108         "image/color":         {"L2"},                // interfaces
 109         "image/color/palette": {"L2", "image/color"},
 110         "reflect":             {"L2"},
 111
 112         "L3": {
 113                 "L2",
 114                 "crypto",
 115                 "crypto/cipher",
 116                 "crypto/subtle",
 117                 "encoding/base32",
 118                 "encoding/base64",
 119                 "encoding/binary",
 120                 "hash",
 121                 "hash/adler32",
 122                 "hash/crc32",
 123                 "hash/crc64",
 124                 "hash/fnv",
 125                 "image",
 126                 "image/color",
 127                 "image/color/palette",
 128                 "reflect",
 129         },
 130
 131         // End of linear dependency definitions.
 132
 133         // Operating system access.
 134         "syscall":                           {"L0", "unicode/utf16"},
 135         "internal/syscall/unix":             {"L0", "syscall"},
 136         "internal/syscall/windows":          {"L0", "syscall"},
 137         "internal/syscall/windows/registry": {"L0", "syscall", "unicode/utf16"},
 138         "time":          {"L0", "syscall", "internal/syscall/windows/registry"},
 139         "os":            {"L1", "os", "syscall", "time", "internal/syscall/windows"},
 140         "path/filepath": {"L2", "os", "syscall"},
 141         "io/ioutil":     {"L2", "os", "path/filepath", "time"},
 142         "os/exec":       {"L2", "os", "path/filepath", "syscall"},
 143         "os/signal":     {"L2", "os", "syscall"},
 144
 145         // OS enables basic operating system functionality,
 146         // but not direct use of package syscall, nor os/signal.
 147         "OS": {
 148                 "io/ioutil",
 149                 "os",
 150                 "os/exec",
 151                 "path/filepath",
 152                 "time",
 153         },
 154
 155         // Formatted I/O: few dependencies (L1) but we must add reflect.
 156         "fmt": {"L1", "os", "reflect"},
 157         "log": {"L1", "os", "fmt", "time"},
 158
 159         // Packages used by testing must be low-level (L2+fmt).
 160         "regexp":         {"L2", "regexp/syntax"},
 161         "regexp/syntax":  {"L2"},
 162         "runtime/debug":  {"L2", "fmt", "io/ioutil", "os", "time"},
 163         "runtime/pprof":  {"L2", "fmt", "text/tabwriter"},
 164         "runtime/trace":  {"L0"},
 165         "text/tabwriter": {"L2"},
 166
 167         "testing":          {"L2", "flag", "fmt", "os", "runtime/pprof", "runtime/trace", "time"},
 168         "testing/iotest":   {"L2", "log"},
 169         "testing/quick":    {"L2", "flag", "fmt", "reflect"},
 170         "internal/testenv": {"L2", "os", "testing"},
 171
 172         // L4 is defined as L3+fmt+log+time, because in general once
 173         // you're using L3 packages, use of fmt, log, or time is not a big deal.
 174         "L4": {
 175                 "L3",
 176                 "fmt",
 177                 "log",
 178                 "time",
 179         },
 180
 181         // Go parser.
 182         "go/ast":     {"L4", "OS", "go/scanner", "go/token"},
 183         "go/doc":     {"L4", "go/ast", "go/token", "regexp", "text/template"},
 184         "go/parser":  {"L4", "OS", "go/ast", "go/scanner", "go/token"},
 185         "go/printer": {"L4", "OS", "go/ast", "go/scanner", "go/token", "text/tabwriter"},
 186         "go/scanner": {"L4", "OS", "go/token"},
 187         "go/token":   {"L4"},
 188
 189         "GOPARSER": {
 190                 "go/ast",
 191                 "go/doc",
 192                 "go/parser",
 193                 "go/printer",
 194                 "go/scanner",
 195                 "go/token",
 196         },
 197
 198         "go/format":       {"L4", "GOPARSER", "internal/format"},
 199         "internal/format": {"L4", "GOPARSER"},
 200
 201         // Go type checking.
 202         "go/constant":               {"L4", "go/token", "math/big"},
 203         "go/importer":               {"L4", "go/internal/gcimporter", "go/internal/gccgoimporter", "go/types"},
 204         "go/internal/gcimporter":    {"L4", "OS", "go/build", "go/constant", "go/token", "go/types", "text/scanner"},
 205         "go/internal/gccgoimporter": {"L4", "OS", "debug/elf", "go/constant", "go/token", "go/types", "text/scanner"},
 206         "go/types":                  {"L4", "GOPARSER", "container/heap", "go/constant"},
 207
 208         // One of a kind.
 209         "archive/tar":              {"L4", "OS", "syscall"},
 210         "archive/zip":              {"L4", "OS", "compress/flate"},
 211         "container/heap":           {"sort"},
 212         "compress/bzip2":           {"L4"},
 213         "compress/flate":           {"L4"},
 214         "compress/gzip":            {"L4", "compress/flate"},
 215         "compress/lzw":             {"L4"},
 216         "compress/zlib":            {"L4", "compress/flate"},
 217         "database/sql":             {"L4", "container/list", "database/sql/driver"},
 218         "database/sql/driver":      {"L4", "time"},
 219         "debug/dwarf":              {"L4"},
 220         "debug/elf":                {"L4", "OS", "debug/dwarf"},
 221         "debug/gosym":              {"L4"},
 222         "debug/macho":              {"L4", "OS", "debug/dwarf"},
 223         "debug/pe":                 {"L4", "OS", "debug/dwarf"},
 224         "debug/plan9obj":           {"L4", "OS"},
 225         "encoding":                 {"L4"},
 226         "encoding/ascii85":         {"L4"},
 227         "encoding/asn1":            {"L4", "math/big"},
 228         "encoding/csv":             {"L4"},
 229         "encoding/gob":             {"L4", "OS", "encoding"},
 230         "encoding/hex":             {"L4"},
 231         "encoding/json":            {"L4", "encoding"},
 232         "encoding/pem":             {"L4"},
 233         "encoding/xml":             {"L4", "encoding"},
 234         "flag":                     {"L4", "OS"},
 235         "go/build":                 {"L4", "OS", "GOPARSER"},
 236         "html":                     {"L4"},
 237         "image/draw":               {"L4", "image/internal/imageutil"},
 238         "image/gif":                {"L4", "compress/lzw", "image/color/palette", "image/draw"},
 239         "image/internal/imageutil": {"L4"},
 240         "image/jpeg":               {"L4", "image/internal/imageutil"},
 241         "image/png":                {"L4", "compress/zlib"},
 242         "index/suffixarray":        {"L4", "regexp"},
 243         "internal/singleflight":    {"sync"},
 244         "internal/trace":           {"L4", "OS"},
 245         "math/big":                 {"L4"},
 246         "mime":                     {"L4", "OS", "syscall", "internal/syscall/windows/registry"},
 247         "mime/quotedprintable":     {"L4"},
 248         "net/internal/socktest":    {"L4", "OS", "syscall"},
 249         "net/url":                  {"L4"},
 250         "text/scanner":             {"L4", "OS"},
 251         "text/template/parse":      {"L4"},
 252
 253         "html/template": {
 254                 "L4", "OS", "encoding/json", "html", "text/template",
 255                 "text/template/parse",
 256         },
 257         "text/template": {
 258                 "L4", "OS", "net/url", "text/template/parse",
 259         },
 260
 261         // Cgo.
 262         "runtime/cgo": {"L0", "C"},
 263         "CGO":         {"C", "runtime/cgo"},
 264
 265         // Fake entry to satisfy the pseudo-import "C"
 266         // that shows up in programs that use cgo.
 267         "C": {},
 268
 269         // Race detector/MSan uses cgo.
 270         "runtime/race": {"C"},
 271         "runtime/msan": {"C"},
 272
 273         // Plan 9 alone needs io/ioutil and os.
 274         "os/user": {"L4", "CGO", "io/ioutil", "os", "syscall"},
 275
 276         // Basic networking.
 277         // Because net must be used by any package that wants to
 278         // do networking portably, it must have a small dependency set: just L0+basic os.
 279         "net": {"L0", "CGO", "math/rand", "os", "sort", "syscall", "time", "internal/syscall/windows", "internal/singleflight"},
 280
 281         // NET enables use of basic network-related packages.
 282         "NET": {
 283                 "net",
 284                 "mime",
 285                 "net/textproto",
 286                 "net/url",
 287         },
 288
 289         // Uses of networking.
 290         "log/syslog":    {"L4", "OS", "net"},
 291         "net/mail":      {"L4", "NET", "OS", "mime"},
 292         "net/textproto": {"L4", "OS", "net"},
 293
 294         // Core crypto.
 295         "crypto/aes":    {"L3"},
 296         "crypto/des":    {"L3"},
 297         "crypto/hmac":   {"L3"},
 298         "crypto/md5":    {"L3"},
 299         "crypto/rc4":    {"L3"},
 300         "crypto/sha1":   {"L3"},
 301         "crypto/sha256": {"L3"},
 302         "crypto/sha512": {"L3"},
 303
 304         "CRYPTO": {
 305                 "crypto/aes",
 306                 "crypto/des",
 307                 "crypto/hmac",
 308                 "crypto/md5",
 309                 "crypto/rc4",
 310                 "crypto/sha1",
 311                 "crypto/sha256",
 312                 "crypto/sha512",
 313         },
 314
 315         // Random byte, number generation.
 316         // This would be part of core crypto except that it imports
 317         // math/big, which imports fmt.
 318         "crypto/rand": {"L4", "CRYPTO", "OS", "math/big", "syscall", "internal/syscall/unix"},
 319
 320         // Mathematical crypto: dependencies on fmt (L4) and math/big.
 321         // We could avoid some of the fmt, but math/big imports fmt anyway.
 322         "crypto/dsa":      {"L4", "CRYPTO", "math/big"},
 323         "crypto/ecdsa":    {"L4", "CRYPTO", "crypto/elliptic", "math/big", "encoding/asn1"},
 324         "crypto/elliptic": {"L4", "CRYPTO", "math/big"},
 325         "crypto/rsa":      {"L4", "CRYPTO", "crypto/rand", "math/big"},
 326
 327         "CRYPTO-MATH": {
 328                 "CRYPTO",
 329                 "crypto/dsa",
 330                 "crypto/ecdsa",
 331                 "crypto/elliptic",
 332                 "crypto/rand",
 333                 "crypto/rsa",
 334                 "encoding/asn1",
 335                 "math/big",
 336         },
 337
 338         // SSL/TLS.
 339         "crypto/tls": {
 340                 "L4", "CRYPTO-MATH", "CGO", "OS",
 341                 "container/list", "crypto/x509", "encoding/pem", "net", "syscall",
 342         },
 343         "crypto/x509": {
 344                 "L4", "CRYPTO-MATH", "OS", "CGO",
 345                 "crypto/x509/pkix", "encoding/pem", "encoding/hex", "net", "syscall",
 346         },
 347         "crypto/x509/pkix": {"L4", "CRYPTO-MATH"},
 348
 349         // Simple net+crypto-aware packages.
 350         "mime/multipart": {"L4", "OS", "mime", "crypto/rand", "net/textproto", "mime/quotedprintable"},
 351         "net/smtp":       {"L4", "CRYPTO", "NET", "crypto/tls"},
 352
 353         // HTTP, kingpin of dependencies.
 354         "net/http": {
 355                 "L4", "NET", "OS",
 356                 "compress/gzip", "crypto/tls", "mime/multipart", "runtime/debug",
 357                 "net/http/internal",
 358                 "golang.org/x/net/http2/hpack",
 359         },
 360         "net/http/internal": {"L4"},
 361
 362         // HTTP-using packages.
 363         "expvar":             {"L4", "OS", "encoding/json", "net/http"},
 364         "net/http/cgi":       {"L4", "NET", "OS", "crypto/tls", "net/http", "regexp"},
 365         "net/http/cookiejar": {"L4", "NET", "net/http"},
 366         "net/http/fcgi":      {"L4", "NET", "OS", "net/http", "net/http/cgi"},
 367         "net/http/httptest":  {"L4", "NET", "OS", "crypto/tls", "flag", "net/http", "net/http/internal"},
 368         "net/http/httputil":  {"L4", "NET", "OS", "net/http", "net/http/internal"},
 369         "net/http/pprof":     {"L4", "OS", "html/template", "net/http", "runtime/pprof", "runtime/trace"},
 370         "net/rpc":            {"L4", "NET", "encoding/gob", "html/template", "net/http"},
 371         "net/rpc/jsonrpc":    {"L4", "NET", "encoding/json", "net/rpc"},
 372 }
 373
 374 // isMacro reports whether p is a package dependency macro
 375 // (uppercase name).
 376 func isMacro(p string) bool {
 377         return 'A' <= p[0] && p[0] <= 'Z'
 378 }
 379
 380 func allowed(pkg string) map[string]bool {
 381         m := map[string]bool{}
 382         var allow func(string)
 383         allow = func(p string) {
 384                 if m[p] {
 385                         return
 386                 }
 387                 m[p] = true // set even for macros, to avoid loop on cycle
 388
 389                 // Upper-case names are macro-expanded.
 390                 if isMacro(p) {
 391                         for _, pp := range pkgDeps[p] {
 392                                 allow(pp)
 393                         }
 394                 }
 395         }
 396         for _, pp := range pkgDeps[pkg] {
 397                 allow(pp)
 398         }
 399         return m
 400 }
 401
 402 var bools = []bool{false, true}
 403 var geese = []string{"android", "darwin", "dragonfly", "freebsd", "linux", "nacl", "netbsd", "openbsd", "plan9", "solaris", "windows"}
 404 var goarches = []string{"386", "amd64", "arm"}
 405
 406 type osPkg struct {
 407         goos, pkg string
 408 }
 409
 410 // allowedErrors are the operating systems and packages known to contain errors
 411 // (currently just "no Go source files")
 412 var allowedErrors = map[osPkg]bool{
 413         osPkg{"windows", "log/syslog"}: true,
 414         osPkg{"plan9", "log/syslog"}:   true,
 415 }
 416
 417 // listStdPkgs returns the same list of packages as "go list std".
 418 func listStdPkgs(goroot string) ([]string, error) {
 419         // Based on cmd/go's matchPackages function.
 420         var pkgs []string
 421
 422         src := filepath.Join(goroot, "src") + string(filepath.Separator)
 423         walkFn := func(path string, fi os.FileInfo, err error) error {
 424                 if err != nil || !fi.IsDir() || path == src {
 425                         return nil
 426                 }
 427
 428                 base := filepath.Base(path)
 429                 if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") || base == "testdata" {
 430                         return filepath.SkipDir
 431                 }
 432
 433                 name := filepath.ToSlash(path[len(src):])
 434                 if name == "builtin" || name == "cmd" || strings.Contains(name, ".") {
 435                         return filepath.SkipDir
 436                 }
 437
 438                 pkgs = append(pkgs, name)
 439                 return nil
 440         }
 441         if err := filepath.Walk(src, walkFn); err != nil {
 442                 return nil, err
 443         }
 444         return pkgs, nil
 445 }
 446
 447 func TestDependencies(t *testing.T) {
 448         iOS := runtime.GOOS == "darwin" && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64")
 449         if runtime.GOOS == "nacl" || iOS {
 450                 // Tests run in a limited file system and we do not
 451                 // provide access to every source file.
 452                 t.Skipf("skipping on %s/%s, missing full GOROOT", runtime.GOOS, runtime.GOARCH)
 453         }
 454
 455         ctxt := Default
 456         all, err := listStdPkgs(ctxt.GOROOT)
 457         if err != nil {
 458                 t.Fatal(err)
 459         }
 460         sort.Strings(all)
 461
 462         for _, pkg := range all {
 463                 imports, err := findImports(pkg)
 464                 if err != nil {
 465                         t.Error(err)
 466                         continue
 467                 }
 468                 ok := allowed(pkg)
 469                 var bad []string
 470                 for _, imp := range imports {
 471                         if !ok[imp] {
 472                                 bad = append(bad, imp)
 473                         }
 474                 }
 475                 if bad != nil {
 476                         t.Errorf("unexpected dependency: %s imports %v", pkg, bad)
 477                 }
 478         }
 479 }
 480
 481 var buildIgnore = []byte("\n// +build ignore")
 482
 483 func findImports(pkg string) ([]string, error) {
 484         dir := filepath.Join(Default.GOROOT, "src", pkg)
 485         files, err := ioutil.ReadDir(dir)
 486         if err != nil {
 487                 return nil, err
 488         }
 489         var imports []string
 490         var haveImport = map[string]bool{}
 491         for _, file := range files {
 492                 name := file.Name()
 493                 if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
 494                         continue
 495                 }
 496                 f, err := os.Open(filepath.Join(dir, name))
 497                 if err != nil {
 498                         return nil, err
 499                 }
 500                 var imp []string
 501                 data, err := readImports(f, false, &imp)
 502                 f.Close()
 503                 if err != nil {
 504                         return nil, fmt.Errorf("reading %v: %v", name, err)
 505                 }
 506                 if bytes.Contains(data, buildIgnore) {
 507                         continue
 508                 }
 509                 for _, quoted := range imp {
 510                         path, err := strconv.Unquote(quoted)
 511                         if err != nil {
 512                                 continue
 513                         }
 514                         if !haveImport[path] {
 515                                 haveImport[path] = true
 516                                 imports = append(imports, path)
 517                         }
 518                 }
 519         }
 520         sort.Strings(imports)
 521         return imports, nil
 522 }