go/types: avoid endless recursion in the Comparable predicate

This is a port of CL 276374 from the dev.typeparams branch. Avoid an
endless recursion in Comparable by tracking types that have already been
considered.

Fixes #43088

Change-Id: I927b29ac544df9bfb5c8c04699d57fafe6cfff73
Reviewed-on: https://go-review.googlesource.com/c/go/+/276552
Run-TryBot: Robert Findley <rfindley@google.com>
Trust: Robert Findley <rfindley@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

diff --git a/src/go/types/issues_test.go b/src/go/types/issues_test.go
index f59f9053979f0614a3b0f8ecc2ce4ea411d9abf0..34850eb03487732119e394218702775a52232f2c 100644 (file)
--- a/src/go/types/issues_test.go
+++ b/src/go/types/issues_test.go
@@ -12,6 +12,7 @@ import (
        "go/ast"
        "go/importer"
        "go/parser"
+       "go/token"
        "internal/testenv"
        "sort"
        "strings"
@@ -523,3 +524,28 @@ func TestIssue34921(t *testing.T) {
                pkg = res // res is imported by the next package in this test
        }
 }
+
+func TestIssue43088(t *testing.T) {
+       // type T1 struct {
+       //         _ T2
+       // }
+       //
+       // type T2 struct {
+       //         _ struct {
+       //                 _ T2
+       //         }
+       // }
+       n1 := NewTypeName(token.NoPos, nil, "T1", nil)
+       T1 := NewNamed(n1, nil, nil)
+       n2 := NewTypeName(token.NoPos, nil, "T2", nil)
+       T2 := NewNamed(n2, nil, nil)
+       s1 := NewStruct([]*Var{NewField(token.NoPos, nil, "_", T2, false)}, nil)
+       T1.SetUnderlying(s1)
+       s2 := NewStruct([]*Var{NewField(token.NoPos, nil, "_", T2, false)}, nil)
+       s3 := NewStruct([]*Var{NewField(token.NoPos, nil, "_", s2, false)}, nil)
+       T2.SetUnderlying(s3)
+
+       // These calls must terminate (no endless recursion).
+       Comparable(T1)
+       Comparable(T2)
+}

diff --git a/src/go/types/predicates.go b/src/go/types/predicates.go
index 057908eacd8da57c3d9c585e2595a84906c509ab..148edbfb767a47d642077d1dbe2c42ed692779f4 100644 (file)
--- a/src/go/types/predicates.go
+++ b/src/go/types/predicates.go
@@ -79,6 +79,18 @@ func IsInterface(typ Type) bool {
 
 // Comparable reports whether values of type T are comparable.
 func Comparable(T Type) bool {
+       return comparable(T, nil)
+}
+
+func comparable(T Type, seen map[Type]bool) bool {
+       if seen[T] {
+               return true
+       }
+       if seen == nil {
+               seen = make(map[Type]bool)
+       }
+       seen[T] = true
+
        switch t := T.Underlying().(type) {
        case *Basic:
                // assume invalid types to be comparable
@@ -88,13 +100,13 @@ func Comparable(T Type) bool {
                return true
        case *Struct:
                for _, f := range t.fields {
-                       if !Comparable(f.typ) {
+                       if !comparable(f.typ, seen) {
                                return false
                        }
                }
                return true
        case *Array:
-               return Comparable(t.elem)
+               return comparable(t.elem, seen)
        }
        return false
 }