src/go/build/deps_test.go

   1 // Copyright 2022 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // This file exercises the import parser but also checks that
   6 // some low-level packages do not have new dependencies added.
   7
   8 package build
   9
  10 import (
  11         "bytes"
  12         "fmt"
  13         "go/token"
  14         "internal/dag"
  15         "internal/testenv"
  16         "io/fs"
  17         "os"
  18         "path/filepath"
  19         "runtime"
  20         "sort"
  21         "strings"
  22         "testing"
  23 )
  24
  25 // depsRules defines the expected dependencies between packages in
  26 // the Go source tree. It is a statement of policy.
  27 //
  28 // DO NOT CHANGE THIS DATA TO FIX BUILDS.
  29 // Existing packages should not have their constraints relaxed
  30 // without prior discussion.
  31 // Negative assertions should almost never be removed.
  32 //
  33 // "a < b" means package b can import package a.
  34 //
  35 // See `go doc internal/dag' for the full syntax.
  36 //
  37 // All-caps names are pseudo-names for specific points
  38 // in the dependency lattice.
  39 var depsRules = `
  40         # No dependencies allowed for any of these packages.
  41         NONE
  42         < constraints, container/list, container/ring,
  43           internal/cfg, internal/coverage, internal/coverage/rtcov,
  44           internal/coverage/uleb128, internal/coverage/calloc,
  45           internal/cpu, internal/goarch,
  46           internal/goexperiment, internal/goos,
  47           internal/goversion, internal/nettrace, internal/platform,
  48           unicode/utf8, unicode/utf16, unicode,
  49           unsafe;
  50
  51         # These packages depend only on internal/goarch and unsafe.
  52         internal/goarch, unsafe
  53         < internal/abi;
  54
  55         # RUNTIME is the core runtime group of packages, all of them very light-weight.
  56         internal/abi, internal/cpu, internal/goarch,
  57         internal/coverage/rtcov, internal/goexperiment,
  58         internal/goos, unsafe
  59         < internal/bytealg
  60         < internal/itoa
  61         < internal/unsafeheader
  62         < runtime/internal/sys
  63         < runtime/internal/syscall
  64         < runtime/internal/atomic
  65         < runtime/internal/math
  66         < runtime
  67         < sync/atomic
  68         < internal/race
  69         < sync
  70         < internal/godebug
  71         < internal/reflectlite
  72         < errors
  73         < internal/oserror, math/bits
  74         < RUNTIME;
  75
  76         RUNTIME
  77         < sort
  78         < container/heap;
  79
  80         RUNTIME
  81         < io;
  82
  83         RUNTIME
  84         < arena;
  85
  86         syscall !< io;
  87         reflect !< sort;
  88
  89         RUNTIME, unicode/utf8
  90         < path;
  91
  92         unicode !< path;
  93
  94         # SYSCALL is RUNTIME plus the packages necessary for basic system calls.
  95         RUNTIME, unicode/utf8, unicode/utf16
  96         < internal/syscall/windows/sysdll, syscall/js
  97         < syscall
  98         < internal/syscall/unix, internal/syscall/windows, internal/syscall/windows/registry
  99         < internal/syscall/execenv
 100         < SYSCALL;
 101
 102         # TIME is SYSCALL plus the core packages about time, including context.
 103         SYSCALL
 104         < time/tzdata
 105         < time
 106         < context
 107         < TIME;
 108
 109         TIME, io, path, sort
 110         < io/fs;
 111
 112         # MATH is RUNTIME plus the basic math packages.
 113         RUNTIME
 114         < math
 115         < MATH;
 116
 117         unicode !< math;
 118
 119         MATH
 120         < math/cmplx;
 121
 122         MATH
 123         < math/rand;
 124
 125         MATH
 126         < runtime/metrics;
 127
 128         MATH, unicode/utf8
 129         < strconv;
 130
 131         unicode !< strconv;
 132
 133         # STR is basic string and buffer manipulation.
 134         RUNTIME, io, unicode/utf8, unicode/utf16, unicode
 135         < bytes, strings
 136         < bufio;
 137
 138         bufio, path, strconv
 139         < STR;
 140
 141         # OS is basic OS access, including helpers (path/filepath, os/exec, etc).
 142         # OS includes string routines, but those must be layered above package os.
 143         # OS does not include reflection.
 144         io/fs
 145         < internal/testlog
 146         < internal/poll
 147         < os
 148         < os/signal;
 149
 150         io/fs
 151         < embed;
 152
 153         unicode, fmt !< net, os, os/signal;
 154
 155         os/signal, STR
 156         < path/filepath
 157         < io/ioutil;
 158
 159         path/filepath, internal/godebug < os/exec;
 160
 161         io/ioutil, os/exec, os/signal
 162         < OS;
 163
 164         reflect !< OS;
 165
 166         OS
 167         < golang.org/x/sys/cpu;
 168
 169         # FMT is OS (which includes string routines) plus reflect and fmt.
 170         # It does not include package log, which should be avoided in core packages.
 171         arena, strconv, unicode
 172         < reflect;
 173
 174         os, reflect
 175         < internal/fmtsort
 176         < fmt;
 177
 178         OS, fmt
 179         < FMT;
 180
 181         log !< FMT;
 182
 183         # Misc packages needing only FMT.
 184         FMT
 185         < html,
 186           internal/dag,
 187           internal/goroot,
 188           internal/types/errors,
 189           mime/quotedprintable,
 190           net/internal/socktest,
 191           net/url,
 192           runtime/trace,
 193           text/scanner,
 194           text/tabwriter;
 195
 196         io, reflect
 197         < internal/saferio;
 198
 199         # encodings
 200         # core ones do not use fmt.
 201         io, strconv
 202         < encoding;
 203
 204         encoding, reflect
 205         < encoding/binary
 206         < encoding/base32, encoding/base64;
 207
 208         FMT, encoding < flag;
 209
 210         fmt !< encoding/base32, encoding/base64;
 211
 212         FMT, encoding/base32, encoding/base64, internal/saferio
 213         < encoding/ascii85, encoding/csv, encoding/gob, encoding/hex,
 214           encoding/json, encoding/pem, encoding/xml, mime;
 215
 216         # hashes
 217         io
 218         < hash
 219         < hash/adler32, hash/crc32, hash/crc64, hash/fnv, hash/maphash;
 220
 221         # math/big
 222         FMT, encoding/binary, math/rand
 223         < math/big;
 224
 225         # compression
 226         FMT, encoding/binary, hash/adler32, hash/crc32
 227         < compress/bzip2, compress/flate, compress/lzw
 228         < archive/zip, compress/gzip, compress/zlib;
 229
 230         # templates
 231         FMT
 232         < text/template/parse;
 233
 234         net/url, text/template/parse
 235         < text/template
 236         < internal/lazytemplate;
 237
 238         encoding/json, html, text/template
 239         < html/template;
 240
 241         # regexp
 242         FMT
 243         < regexp/syntax
 244         < regexp
 245         < internal/lazyregexp;
 246
 247         # suffix array
 248         encoding/binary, regexp
 249         < index/suffixarray;
 250
 251         # executable parsing
 252         FMT, encoding/binary, compress/zlib, internal/saferio
 253         < runtime/debug
 254         < debug/dwarf
 255         < debug/elf, debug/gosym, debug/macho, debug/pe, debug/plan9obj, internal/xcoff
 256         < debug/buildinfo
 257         < DEBUG;
 258
 259         # go parser and friends.
 260         FMT
 261         < go/token
 262         < go/scanner
 263         < go/ast
 264         < go/internal/typeparams
 265         < go/parser;
 266
 267         FMT
 268         < go/build/constraint, go/doc/comment;
 269
 270         go/build/constraint, go/doc/comment, go/parser, text/tabwriter
 271         < go/printer
 272         < go/format;
 273
 274         go/doc/comment, go/parser, internal/lazyregexp, text/template
 275         < go/doc;
 276
 277         math/big, go/token
 278         < go/constant;
 279
 280         container/heap, go/constant, go/parser, internal/types/errors, regexp
 281         < go/types;
 282
 283         FMT, internal/goexperiment
 284         < internal/buildcfg;
 285
 286         go/build/constraint, go/doc, go/parser, internal/buildcfg, internal/goroot, internal/goversion
 287         < go/build;
 288
 289         # databases
 290         FMT
 291         < database/sql/internal
 292         < database/sql/driver
 293         < database/sql;
 294
 295         # images
 296         FMT, compress/lzw, compress/zlib
 297         < image/color
 298         < image, image/color/palette
 299         < image/internal/imageutil
 300         < image/draw
 301         < image/gif, image/jpeg, image/png;
 302
 303         # cgo, delayed as long as possible.
 304         # If you add a dependency on CGO, you must add the package
 305         # to cgoPackages in cmd/dist/test.go as well.
 306         RUNTIME
 307         < C
 308         < runtime/cgo
 309         < CGO
 310         < runtime/msan, runtime/asan;
 311
 312         # runtime/race
 313         NONE < runtime/race/internal/amd64v1;
 314         NONE < runtime/race/internal/amd64v3;
 315         CGO, runtime/race/internal/amd64v1, runtime/race/internal/amd64v3 < runtime/race;
 316
 317         # Bulk of the standard library must not use cgo.
 318         # The prohibition stops at net and os/user.
 319         C !< fmt, go/types, CRYPTO-MATH;
 320
 321         CGO, OS
 322         < plugin;
 323
 324         CGO, FMT
 325         < os/user
 326         < archive/tar;
 327
 328         sync
 329         < internal/singleflight;
 330
 331         os
 332         < golang.org/x/net/dns/dnsmessage,
 333           golang.org/x/net/lif,
 334           golang.org/x/net/route;
 335
 336         os, runtime, strconv, sync, unsafe,
 337         internal/godebug
 338         < internal/intern;
 339
 340         internal/bytealg, internal/intern, internal/itoa, math/bits, sort, strconv
 341         < net/netip;
 342
 343         # net is unavoidable when doing any networking,
 344         # so large dependencies must be kept out.
 345         # This is a long-looking list but most of these
 346         # are small with few dependencies.
 347         CGO,
 348         golang.org/x/net/dns/dnsmessage,
 349         golang.org/x/net/lif,
 350         golang.org/x/net/route,
 351         internal/godebug,
 352         internal/nettrace,
 353         internal/poll,
 354         internal/singleflight,
 355         internal/race,
 356         net/netip,
 357         os
 358         < net;
 359
 360         fmt, unicode !< net;
 361         math/rand !< net; # net uses runtime instead
 362
 363         # NET is net plus net-helper packages.
 364         FMT, net
 365         < net/textproto;
 366
 367         mime, net/textproto, net/url
 368         < NET;
 369
 370         # logging - most packages should not import; http and up is allowed
 371         FMT
 372         < log;
 373
 374         log !< crypto/tls, database/sql, go/importer, testing;
 375
 376         FMT, log, net
 377         < log/syslog;
 378
 379         NET, log
 380         < net/mail;
 381
 382         NONE < crypto/internal/boring/sig, crypto/internal/boring/syso;
 383         sync/atomic < crypto/internal/boring/bcache, crypto/internal/boring/fipstls;
 384         crypto/internal/boring/sig, crypto/internal/boring/fipstls < crypto/tls/fipsonly;
 385
 386         # CRYPTO is core crypto algorithms - no cgo, fmt, net.
 387         # Unfortunately, stuck with reflect via encoding/binary.
 388         crypto/internal/boring/sig,
 389         crypto/internal/boring/syso,
 390         encoding/binary,
 391         golang.org/x/sys/cpu,
 392         hash, embed
 393         < crypto
 394         < crypto/subtle
 395         < crypto/internal/alias
 396         < crypto/cipher;
 397
 398         crypto/cipher,
 399         crypto/internal/boring/bcache
 400         < crypto/internal/boring
 401         < crypto/boring;
 402
 403         crypto/internal/alias
 404         < crypto/internal/randutil
 405         < crypto/internal/nistec/fiat
 406         < crypto/internal/nistec
 407         < crypto/internal/edwards25519/field
 408         < crypto/internal/edwards25519;
 409
 410         crypto/boring
 411         < crypto/aes, crypto/des, crypto/hmac, crypto/md5, crypto/rc4,
 412           crypto/sha1, crypto/sha256, crypto/sha512;
 413
 414         crypto/boring, crypto/internal/edwards25519/field
 415         < crypto/ecdh;
 416
 417         crypto/aes,
 418         crypto/des,
 419         crypto/ecdh,
 420         crypto/hmac,
 421         crypto/internal/edwards25519,
 422         crypto/md5,
 423         crypto/rc4,
 424         crypto/sha1,
 425         crypto/sha256,
 426         crypto/sha512
 427         < CRYPTO;
 428
 429         CGO, fmt, net !< CRYPTO;
 430
 431         # CRYPTO-MATH is core bignum-based crypto - no cgo, net; fmt now ok.
 432         CRYPTO, FMT, math/big
 433         < crypto/internal/boring/bbig
 434         < crypto/rand
 435         < crypto/ed25519
 436         < encoding/asn1
 437         < golang.org/x/crypto/cryptobyte/asn1
 438         < golang.org/x/crypto/cryptobyte
 439         < crypto/dsa, crypto/elliptic, crypto/rsa
 440         < crypto/ecdsa
 441         < CRYPTO-MATH;
 442
 443         CGO, net !< CRYPTO-MATH;
 444
 445         # TLS, Prince of Dependencies.
 446         CRYPTO-MATH, NET, container/list, encoding/hex, encoding/pem
 447         < golang.org/x/crypto/internal/alias
 448         < golang.org/x/crypto/internal/subtle
 449         < golang.org/x/crypto/chacha20
 450         < golang.org/x/crypto/internal/poly1305
 451         < golang.org/x/crypto/chacha20poly1305
 452         < golang.org/x/crypto/hkdf
 453         < crypto/x509/internal/macos
 454         < crypto/x509/pkix;
 455
 456         crypto/internal/boring/fipstls, crypto/x509/pkix
 457         < crypto/x509
 458         < crypto/tls;
 459
 460         # crypto-aware packages
 461
 462         DEBUG, go/build, go/types, text/scanner, crypto/md5
 463         < internal/pkgbits
 464         < go/internal/gcimporter, go/internal/gccgoimporter, go/internal/srcimporter
 465         < go/importer;
 466
 467         NET, crypto/rand, mime/quotedprintable
 468         < mime/multipart;
 469
 470         crypto/tls
 471         < net/smtp;
 472
 473         # HTTP, King of Dependencies.
 474
 475         FMT
 476         < golang.org/x/net/http2/hpack
 477         < net/http/internal, net/http/internal/ascii, net/http/internal/testcert;
 478
 479         FMT, NET, container/list, encoding/binary, log
 480         < golang.org/x/text/transform
 481         < golang.org/x/text/unicode/norm
 482         < golang.org/x/text/unicode/bidi
 483         < golang.org/x/text/secure/bidirule
 484         < golang.org/x/net/idna
 485         < golang.org/x/net/http/httpguts, golang.org/x/net/http/httpproxy;
 486
 487         NET, crypto/tls
 488         < net/http/httptrace;
 489
 490         compress/gzip,
 491         golang.org/x/net/http/httpguts,
 492         golang.org/x/net/http/httpproxy,
 493         golang.org/x/net/http2/hpack,
 494         net/http/internal,
 495         net/http/internal/ascii,
 496         net/http/internal/testcert,
 497         net/http/httptrace,
 498         mime/multipart,
 499         log
 500         < net/http;
 501
 502         # HTTP-aware packages
 503
 504         encoding/json, net/http
 505         < expvar;
 506
 507         net/http, net/http/internal/ascii
 508         < net/http/cookiejar, net/http/httputil;
 509
 510         net/http, flag
 511         < net/http/httptest;
 512
 513         net/http, regexp
 514         < net/http/cgi
 515         < net/http/fcgi;
 516
 517         # Profiling
 518         FMT, compress/gzip, encoding/binary, text/tabwriter
 519         < runtime/pprof;
 520
 521         OS, compress/gzip, regexp
 522         < internal/profile;
 523
 524         html, internal/profile, net/http, runtime/pprof, runtime/trace
 525         < net/http/pprof;
 526
 527         # RPC
 528         encoding/gob, encoding/json, go/token, html/template, net/http
 529         < net/rpc
 530         < net/rpc/jsonrpc;
 531
 532         # System Information
 533         internal/cpu, sync
 534         < internal/sysinfo;
 535
 536         # Test-only
 537         log
 538         < testing/iotest
 539         < testing/fstest;
 540
 541         FMT, flag, math/rand
 542         < testing/quick;
 543
 544         FMT, DEBUG, flag, runtime/trace, internal/sysinfo, math/rand
 545         < testing;
 546
 547         FMT, crypto/sha256, encoding/json, go/ast, go/parser, go/token,
 548         internal/godebug, math/rand, encoding/hex, crypto/sha256
 549         < internal/fuzz;
 550
 551         internal/fuzz, internal/testlog, runtime/pprof, regexp
 552         < testing/internal/testdeps;
 553
 554         OS, flag, testing, internal/cfg, internal/platform, internal/goroot
 555         < internal/testenv;
 556
 557         OS, encoding/base64
 558         < internal/obscuretestdata;
 559
 560         CGO, OS, fmt
 561         < internal/testpty;
 562
 563         NET, testing, math/rand
 564         < golang.org/x/net/nettest;
 565
 566         syscall
 567         < os/exec/internal/fdtest;
 568
 569         FMT, container/heap, math/rand
 570         < internal/trace;
 571
 572         FMT
 573         < internal/diff, internal/txtar;
 574
 575         FMT, crypto/md5, encoding/binary, regexp, sort, text/tabwriter, unsafe,
 576         internal/coverage, internal/coverage/uleb128
 577         < internal/coverage/cmerge,
 578           internal/coverage/pods,
 579           internal/coverage/slicereader,
 580           internal/coverage/slicewriter;
 581
 582         internal/coverage/slicereader, internal/coverage/slicewriter
 583         < internal/coverage/stringtab
 584         < internal/coverage/decodecounter, internal/coverage/decodemeta,
 585           internal/coverage/encodecounter, internal/coverage/encodemeta;
 586
 587         internal/coverage/cmerge
 588         < internal/coverage/cformat;
 589
 590         runtime/debug,
 591         internal/coverage/calloc,
 592         internal/coverage/cformat,
 593         internal/coverage/decodecounter, internal/coverage/decodemeta,
 594         internal/coverage/encodecounter, internal/coverage/encodemeta,
 595         internal/coverage/pods
 596         < runtime/coverage;
 597 `
 598
 599 // listStdPkgs returns the same list of packages as "go list std".
 600 func listStdPkgs(goroot string) ([]string, error) {
 601         // Based on cmd/go's matchPackages function.
 602         var pkgs []string
 603
 604         src := filepath.Join(goroot, "src") + string(filepath.Separator)
 605         walkFn := func(path string, d fs.DirEntry, err error) error {
 606                 if err != nil || !d.IsDir() || path == src {
 607                         return nil
 608                 }
 609
 610                 base := filepath.Base(path)
 611                 if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") || base == "testdata" {
 612                         return filepath.SkipDir
 613                 }
 614
 615                 name := filepath.ToSlash(path[len(src):])
 616                 if name == "builtin" || name == "cmd" {
 617                         return filepath.SkipDir
 618                 }
 619
 620                 pkgs = append(pkgs, strings.TrimPrefix(name, "vendor/"))
 621                 return nil
 622         }
 623         if err := filepath.WalkDir(src, walkFn); err != nil {
 624                 return nil, err
 625         }
 626         return pkgs, nil
 627 }
 628
 629 func TestDependencies(t *testing.T) {
 630         if !testenv.HasSrc() {
 631                 // Tests run in a limited file system and we do not
 632                 // provide access to every source file.
 633                 t.Skipf("skipping on %s/%s, missing full GOROOT", runtime.GOOS, runtime.GOARCH)
 634         }
 635
 636         ctxt := Default
 637         all, err := listStdPkgs(ctxt.GOROOT)
 638         if err != nil {
 639                 t.Fatal(err)
 640         }
 641         sort.Strings(all)
 642
 643         sawImport := map[string]map[string]bool{} // from package => to package => true
 644         policy := depsPolicy(t)
 645
 646         for _, pkg := range all {
 647                 imports, err := findImports(pkg)
 648                 if err != nil {
 649                         t.Error(err)
 650                         continue
 651                 }
 652                 if sawImport[pkg] == nil {
 653                         sawImport[pkg] = map[string]bool{}
 654                 }
 655                 var bad []string
 656                 for _, imp := range imports {
 657                         sawImport[pkg][imp] = true
 658                         if !policy.HasEdge(pkg, imp) {
 659                                 bad = append(bad, imp)
 660                         }
 661                 }
 662                 if bad != nil {
 663                         t.Errorf("unexpected dependency: %s imports %v", pkg, bad)
 664                 }
 665         }
 666 }
 667
 668 var buildIgnore = []byte("\n//go:build ignore")
 669
 670 func findImports(pkg string) ([]string, error) {
 671         vpkg := pkg
 672         if strings.HasPrefix(pkg, "golang.org") {
 673                 vpkg = "vendor/" + pkg
 674         }
 675         dir := filepath.Join(Default.GOROOT, "src", vpkg)
 676         files, err := os.ReadDir(dir)
 677         if err != nil {
 678                 return nil, err
 679         }
 680         var imports []string
 681         var haveImport = map[string]bool{}
 682         if pkg == "crypto/internal/boring" {
 683                 haveImport["C"] = true // kludge: prevent C from appearing in crypto/internal/boring imports
 684         }
 685         fset := token.NewFileSet()
 686         for _, file := range files {
 687                 name := file.Name()
 688                 if name == "slice_go14.go" || name == "slice_go18.go" {
 689                         // These files are for compiler bootstrap with older versions of Go and not built in the standard build.
 690                         continue
 691                 }
 692                 if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
 693                         continue
 694                 }
 695                 info := fileInfo{
 696                         name: filepath.Join(dir, name),
 697                         fset: fset,
 698                 }
 699                 f, err := os.Open(info.name)
 700                 if err != nil {
 701                         return nil, err
 702                 }
 703                 err = readGoInfo(f, &info)
 704                 f.Close()
 705                 if err != nil {
 706                         return nil, fmt.Errorf("reading %v: %v", name, err)
 707                 }
 708                 if info.parsed.Name.Name == "main" {
 709                         continue
 710                 }
 711                 if bytes.Contains(info.header, buildIgnore) {
 712                         continue
 713                 }
 714                 for _, imp := range info.imports {
 715                         path := imp.path
 716                         if !haveImport[path] {
 717                                 haveImport[path] = true
 718                                 imports = append(imports, path)
 719                         }
 720                 }
 721         }
 722         sort.Strings(imports)
 723         return imports, nil
 724 }
 725
 726 // depsPolicy returns a map m such that m[p][d] == true when p can import d.
 727 func depsPolicy(t *testing.T) *dag.Graph {
 728         g, err := dag.Parse(depsRules)
 729         if err != nil {
 730                 t.Fatal(err)
 731         }
 732         return g
 733 }
 734
 735 // TestStdlibLowercase tests that all standard library package names are
 736 // lowercase. See Issue 40065.
 737 func TestStdlibLowercase(t *testing.T) {
 738         if !testenv.HasSrc() {
 739                 t.Skipf("skipping on %s/%s, missing full GOROOT", runtime.GOOS, runtime.GOARCH)
 740         }
 741
 742         ctxt := Default
 743         all, err := listStdPkgs(ctxt.GOROOT)
 744         if err != nil {
 745                 t.Fatal(err)
 746         }
 747
 748         for _, pkgname := range all {
 749                 if strings.ToLower(pkgname) != pkgname {
 750                         t.Errorf("package %q should not use upper-case path", pkgname)
 751                 }
 752         }
 753 }
 754
 755 // TestFindImports tests that findImports works.  See #43249.
 756 func TestFindImports(t *testing.T) {
 757         imports, err := findImports("go/build")
 758         if err != nil {
 759                 t.Fatal(err)
 760         }
 761         t.Logf("go/build imports %q", imports)
 762         want := []string{"bytes", "os", "path/filepath", "strings"}
 763 wantLoop:
 764         for _, w := range want {
 765                 for _, imp := range imports {
 766                         if imp == w {
 767                                 continue wantLoop
 768                         }
 769                 }
 770                 t.Errorf("expected to find %q in import list", w)
 771         }
 772 }