Skip to content

Commit

Permalink
feat: improve parsing (elastic#51)
Browse files Browse the repository at this point in the history 
This simplifies the complexity of the processing by having separate `processType`, `loadType` and `loadAliasType`funcs. This also removes ArrayKind, as it is not supported by controller-gen. And this fixes the parsing of underlying types required to correctly parse and propagate markers.

---------
Signed-off-by: Amund Tenstad <github@amund.io>
  • Loading branch information
@tenstad
tenstad committed Oct 6, 2023
1 parent c9060e9 commit 365ecbe
Show file tree
Hide file tree
Showing 10 changed files with 257 additions and 218 deletions.
292 changes: 117 additions & 175 deletions processor/processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func Process(config *config.Config) ([]types.GroupVersionDetails, error) {

details.Types = make(types.TypeMap)
for name, t := range gvi.types {
key := types.Key(t)
key := types.Identifier(t)

if p.shouldIgnoreType(key) {
zap.S().Debugw("Skipping excluded type", "type", name)
Expand Down Expand Up @@ -192,10 +192,9 @@ func (p *processor) findAPITypes(directory string) error {
key := fmt.Sprintf("%s.%s", pkg.PkgPath, info.Name)
typeDef, ok := p.types[key]
if !ok {
typeDef = p.processType(pkg, info, 0)
typeDef = p.processType(pkg, nil, pkg.TypesInfo.TypeOf(info.RawSpec.Name), 0)
}

p.types[key] = typeDef
if typeDef != nil && typeDef.Kind != types.BasicKind {
gvInfo.types[info.Name] = typeDef
}
Expand Down Expand Up @@ -264,47 +263,116 @@ func (p *processor) extractPkgDocumentation(pkg *loader.Package) string {
return strings.Join(pkgComments, "\n")
}

func (p *processor) processType(pkg *loader.Package, info *markers.TypeInfo, depth int) *types.Type {
typeDef := &types.Type{
Name: info.Name,
Package: pkg.PkgPath,
Doc: info.Doc,
}
func (p *processor) processType(pkg *loader.Package, parentType *types.Type, t gotypes.Type, depth int) *types.Type {
typeDef := mkType(pkg, t)

if p.useRawDocstring && info.RawDecl != nil {
// use raw docstring to support multi-line and indent preservation
typeDef.Doc = strings.TrimSuffix(info.RawDecl.Doc.Text(), "\n")
if processed, ok := p.types[typeDef.UID]; ok {
return processed
}

// if the field list is non-empty, this is a struct
if len(info.Fields) > 0 {
typeDef.Kind = types.StructKind
return p.processStructFields(typeDef, pkg, info, depth)
info := p.parser.LookupType(pkg, typeDef.Name)
if info != nil {
typeDef.Doc = info.Doc

if p.useRawDocstring && info.RawDecl != nil {
// use raw docstring to support multi-line and indent preservation
typeDef.Doc = strings.TrimSuffix(info.RawDecl.Doc.Text(), "\n")
}
}

t := pkg.TypesInfo.TypeOf(info.RawSpec.Name)
if t == nil {
zap.S().Warnw("Failed to determine AST type", "package", pkg.PkgPath, "type", info.Name)
if depth > p.maxDepth {
zap.S().Debugw("Not loading type due to reaching max recursion depth", "type", t.String())
typeDef.Kind = types.UnknownKind
return typeDef
}

tmpType := p.loadType(pkg, t, depth)
if tmpType == nil {
zap.S().Debugw("Load", "package", typeDef.Package, "name", typeDef.Name)

switch t := t.(type) {
case nil:
typeDef.Kind = types.UnknownKind
return typeDef
zap.S().Warnw("Failed to determine AST type", "package", pkg.PkgPath, "type", t.String())

case *gotypes.Named:
// Import the type's package if not within current package
if typeDef.Package != pkg.PkgPath {
imports := pkg.Imports()
importPkg, ok := imports[typeDef.Package]
if !ok {
zap.S().Warnw("Imported type cannot be found", "name", typeDef.Name, "package", typeDef.Package)
return typeDef
}
p.parser.NeedPackage(importPkg)
pkg = importPkg
}

typeDef.Kind = types.AliasKind
underlying := t.Underlying()
info := p.parser.LookupType(pkg, typeDef.Name)
if info != nil {
underlying = pkg.TypesInfo.TypeOf(info.RawSpec.Type)
}
typeDef.UnderlyingType = p.processType(pkg, typeDef, underlying, depth+1)
p.addReference(typeDef, typeDef.UnderlyingType)

case *gotypes.Struct:
if parentType != nil {
// Rather than the parent being a Named type with a "raw" Struct as
// UnderlyingType, convert the parent to a Struct type.
parentType.Kind = types.StructKind
if info := p.parser.LookupType(pkg, parentType.Name); info != nil {
p.processStructFields(parentType, pkg, info, depth)
}
// Abort processing type and return nil as UnderlyingType of parent.
return nil
} else {
zap.S().Warnw("Anonymous structs are not supported", "package", pkg.PkgPath, "type", t.String())
typeDef.Name = ""
typeDef.Package = ""
typeDef.Kind = types.UnsupportedKind
}

case *gotypes.Pointer:
typeDef.Kind = types.PointerKind
typeDef.UnderlyingType = p.processType(pkg, typeDef, t.Elem(), depth+1)
if typeDef.UnderlyingType != nil {
typeDef.Package = typeDef.UnderlyingType.Package
}

case *gotypes.Slice:
typeDef.Kind = types.SliceKind
typeDef.UnderlyingType = p.processType(pkg, typeDef, t.Elem(), depth+1)
if typeDef.UnderlyingType != nil {
typeDef.Package = typeDef.UnderlyingType.Package
}

case *gotypes.Map:
typeDef.Kind = types.MapKind
typeDef.KeyType = p.processType(pkg, typeDef, t.Key(), depth+1)
typeDef.ValueType = p.processType(pkg, typeDef, t.Elem(), depth+1)
if typeDef.ValueType != nil {
typeDef.Package = typeDef.ValueType.Package
}

case *gotypes.Basic:
typeDef.Kind = types.BasicKind
typeDef.Package = ""

case *gotypes.Interface:
typeDef.Kind = types.InterfaceKind

default:
typeDef.Kind = types.UnsupportedKind
}

tmpType.Name = typeDef.Name
tmpType.Package = typeDef.Package
tmpType.Doc = typeDef.Doc
return tmpType
p.types[typeDef.UID] = typeDef
return typeDef
}

func (p *processor) processStructFields(parentType *types.Type, pkg *loader.Package, info *markers.TypeInfo, depth int) *types.Type {
func (p *processor) processStructFields(parentType *types.Type, pkg *loader.Package, info *markers.TypeInfo, depth int) {
logger := zap.S().With("package", pkg.PkgPath, "type", parentType.String())
logger.Debugw("Processing struct fields")
parentTypeKey := types.Key(parentType)
parentTypeKey := types.Identifier(parentType)

for _, f := range info.Fields {
fieldDef := &types.Field{
Expand All @@ -330,11 +398,14 @@ func (p *processor) processStructFields(parentType *types.Type, pkg *loader.Pack
}

logger.Debugw("Loading field type", "field", fieldDef.Name)
if fieldDef.Type = p.loadType(pkg, t, depth); fieldDef.Type == nil {
if fieldDef.Type = p.processType(pkg, nil, t, depth); fieldDef.Type == nil {
logger.Debugw("Failed to load type for field", "field", f.Name, "type", t.String())
continue
}

// Keep old behaviour, where struct fields are never regarded as imported
fieldDef.Type.Imported = false

if fieldDef.Name == "" {
fieldDef.Name = fieldDef.Type.Name
}
Expand All @@ -347,106 +418,22 @@ func (p *processor) processStructFields(parentType *types.Type, pkg *loader.Pack
parentType.Fields = append(parentType.Fields, fieldDef)
p.addReference(parentType, fieldDef.Type)
}

return parentType
}

func (p *processor) loadType(pkg *loader.Package, t gotypes.Type, depth int) *types.Type {
if depth > p.maxDepth {
zap.S().Debugw("Not loading type due to reaching max recursion depth", "type", t.String())
return nil
}

typeDef := mkType(pkg, t)

zap.S().Debugw("Load", "package", typeDef.Package, "name", typeDef.Name)

switch x := t.(type) {
case *gotypes.Pointer:
if y, ok := p.types[types.Key(typeDef)]; ok {
typeDef = y.Copy()
typeDef.UnderlyingType = y
} else {
typeDef.UnderlyingType = p.loadType(pkg, x.Elem(), depth+1)
}
typeDef.Kind = types.PointerKind
if typeDef.UnderlyingType != nil && typeDef.UnderlyingType.Kind == types.BasicKind {
typeDef.Package = ""
}
return typeDef

case *gotypes.Slice:
if y, ok := p.types[types.Key(typeDef)]; ok {
typeDef = y.Copy()
typeDef.UnderlyingType = y
} else {
typeDef.UnderlyingType = p.loadType(pkg, x.Elem(), depth+1)
}
typeDef.Kind = types.SliceKind
if typeDef.UnderlyingType != nil && typeDef.UnderlyingType.Kind == types.BasicKind {
typeDef.Package = ""
}
return typeDef

case *gotypes.Array:
if y, ok := p.types[types.Key(typeDef)]; ok {
typeDef = y.Copy()
typeDef.UnderlyingType = y
} else {
typeDef.UnderlyingType = p.loadType(pkg, x.Elem(), depth+1)
}
typeDef.Kind = types.ArrayKind
if typeDef.UnderlyingType != nil && typeDef.UnderlyingType.Kind == types.BasicKind {
typeDef.Package = ""
}
return typeDef
}

if x, ok := p.types[types.Key(typeDef)]; ok {
return x
}

switch x := t.(type) {
case *gotypes.Basic:
typeDef.Kind = types.BasicKind
typeDef.Package = ""

case *gotypes.Map:
typeDef.Kind = types.MapKind
typeDef.KeyType = p.loadType(pkg, x.Key(), depth+1)
typeDef.ValueType = p.loadType(pkg, x.Elem(), depth+1)

case *gotypes.Named:
typeDef.Kind = types.AliasKind
typeDef = p.loadAliasType(typeDef, pkg, x.Underlying(), depth)

case *gotypes.Interface:
if x.Empty() {
typeDef.Kind = types.InterfaceKind
} else {
return nil
}

default:
return nil
}

p.types[types.Key(typeDef)] = typeDef

return typeDef
}

func mkType(pkg *loader.Package, t gotypes.Type) *types.Type {
qualifier := gotypes.RelativeTo(pkg.Types)
cleanTypeName := strings.TrimLeft(gotypes.TypeString(t, qualifier), "*[]")

typeDef := &types.Type{
UID: t.String(),
Name: cleanTypeName,
Package: pkg.PkgPath,
}

// is this is an imported type?
if dotPos := strings.LastIndexByte(cleanTypeName, '.'); dotPos >= 0 {
// Check if the type is imported
rawType := strings.HasPrefix(cleanTypeName, "struct{") || strings.HasPrefix(cleanTypeName, "interface{")
dotPos := strings.LastIndexByte(cleanTypeName, '.')
if !rawType && dotPos >= 0 {
typeDef.Name = cleanTypeName[dotPos+1:]
typeDef.Package = cleanTypeName[:dotPos]
typeDef.Imported = true
Expand All @@ -455,76 +442,31 @@ func mkType(pkg *loader.Package, t gotypes.Type) *types.Type {
return typeDef
}

func (p *processor) loadAliasType(typeDef *types.Type, pkg *loader.Package, underlying gotypes.Type, depth int) *types.Type {
tPkg := pkg

// check whether this type is imported
if typeDef.Package != pkg.PkgPath {
imports := pkg.Imports()
importPkg, ok := imports[typeDef.Package]
if !ok {
zap.S().Warnw("Imported type cannot be found", "name", typeDef.Name, "package", typeDef.Package)
return typeDef
}

p.parser.NeedPackage(importPkg)
tPkg = importPkg
}

// find the type from the parser
tInfo := p.parser.LookupType(tPkg, typeDef.Name)
if tInfo == nil {
zap.S().Warnw("Failed to find type", "name", typeDef.Name, "package", typeDef.Package)
return typeDef
}

if bt, ok := underlying.(*gotypes.Basic); ok {
typeDef.UnderlyingType = &types.Type{Name: bt.String(), Kind: types.BasicKind}
typeDef.Doc = tInfo.Doc
return typeDef
}

return p.processType(tPkg, tInfo, depth+1)
}

// Every child that has a reference to 'originalType', will also get a reference to 'additionalType'.
func (p *processor) propagateReference(originalType *types.Type, additionalType *types.Type) {
originalTypeKey := types.Key(originalType)
additionalTypeKey := types.Key(additionalType)

for _, parentRefs := range p.references {
if _, ok := parentRefs[originalTypeKey]; ok {
parentRefs[additionalTypeKey] = struct{}{}
if _, ok := parentRefs[originalType.UID]; ok {
parentRefs[additionalType.UID] = struct{}{}
}
}
}

func (p *processor) addReference(parent *types.Type, child *types.Type) {
if child == nil || child.Kind == types.BasicKind {
if child == nil {
return
}

addRef := func(t *types.Type) {
if t == nil || t.Kind == types.BasicKind {
return
}

parentTypeKey := types.Key(parent)
childTypeKey := types.Key(t)
if p.references[childTypeKey] == nil {
p.references[childTypeKey] = make(map[string]struct{})
}
p.references[childTypeKey][parentTypeKey] = struct{}{}
}

switch child.Kind {
case types.AliasKind, types.StructKind:
addRef(child)
case types.ArrayKind, types.SliceKind, types.PointerKind:
addRef(child.UnderlyingType)
case types.SliceKind, types.PointerKind:
p.addReference(parent, child.UnderlyingType)
case types.MapKind:
addRef(child.KeyType)
addRef(child.ValueType)
p.addReference(parent, child.KeyType)
p.addReference(parent, child.ValueType)
case types.AliasKind, types.StructKind:
if p.references[child.UID] == nil {
p.references[child.UID] = make(map[string]struct{})
}
p.references[child.UID][parent.UID] = struct{}{}
}
}

Expand Down
2 changes: 1 addition & 1 deletion renderer/asciidoctor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func (adr *AsciidoctorRenderer) RenderType(t *types.Type) string {
sb.WriteString(", values:")
sb.WriteString(adr.RenderTypeLink(t.ValueType))
sb.WriteString(")")
case types.ArrayKind, types.SliceKind:
case types.SliceKind:
sb.WriteString(adr.RenderTypeLink(t.UnderlyingType))
sb.WriteString(" array")
default:
Expand Down
Loading

0 comments on commit 365ecbe

Please sign in to comment.