Skip to content

Commit

Permalink
DEVPROD-9648 add alias check run validation (#8334)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ablack12
ablack12 authored Oct 3, 2024
1 parent a9c26a0 commit af843d2
Show file tree
Hide file tree
Showing 4 changed files with 199 additions and 65 deletions.
32 changes: 16 additions & 16 deletions model/project.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1776,9 +1776,9 @@ func (p *Project) ResolvePatchVTs(patchDoc *patch.Patch, requester, alias string
aliases, err := findAliasesForPatch(p.Identifier, alias, patchDoc)
catcher.Wrapf(err, "retrieving alias '%s' for patched project config '%s'", alias, patchDoc.Id.Hex())

var aliasPairs, displayTaskPairs []TVPair
aliasPairs := TaskVariantPairs{}
if !catcher.HasErrors() {
aliasPairs, displayTaskPairs, err = p.BuildProjectTVPairsWithAlias(aliases, requester)
aliasPairs, err = p.BuildProjectTVPairsWithAlias(aliases, requester)
catcher.Wrap(err, "getting task/variant pairs for alias")
}
grip.Error(message.WrapError(catcher.Resolve(), message.Fields{
Expand All @@ -1788,8 +1788,8 @@ func (p *Project) ResolvePatchVTs(patchDoc *patch.Patch, requester, alias string
}))

if !catcher.HasErrors() {
pairs.ExecTasks = append(pairs.ExecTasks, aliasPairs...)
pairs.DisplayTasks = append(pairs.DisplayTasks, displayTaskPairs...)
pairs.ExecTasks = append(pairs.ExecTasks, aliasPairs.ExecTasks...)
pairs.DisplayTasks = append(pairs.DisplayTasks, aliasPairs.DisplayTasks...)
}
}

Expand All @@ -1808,7 +1808,7 @@ func (p *Project) ResolvePatchVTs(patchDoc *patch.Patch, requester, alias string
return bvs, tasks, vts
}

// GetVariantTasks returns all the build variants and all tasks specified for
// GetAllVariantTasks returns all the build variants and all tasks specified for
// each build variant.
func (p *Project) GetAllVariantTasks() []patch.VariantTasks {
var vts []patch.VariantTasks
Expand Down Expand Up @@ -1961,20 +1961,22 @@ func (p *Project) extractDisplayTasks(pairs TaskVariantPairs) TaskVariantPairs {
// BuildProjectTVPairsWithAlias returns variants and tasks for a project alias.
// This filters out tasks that cannot run due to being disabled or having an
// unmatched requester (e.g. a patch-only task for a mainline commit).
func (p *Project) BuildProjectTVPairsWithAlias(aliases []ProjectAlias, requester string) ([]TVPair, []TVPair, error) {
pairs := []TVPair{}
displayTaskPairs := []TVPair{}
func (p *Project) BuildProjectTVPairsWithAlias(aliases []ProjectAlias, requester string) (TaskVariantPairs, error) {
res := TaskVariantPairs{
ExecTasks: []TVPair{},
DisplayTasks: []TVPair{},
}
for _, alias := range aliases {
var variantRegex *regexp.Regexp
variantRegex, err := alias.getVariantRegex()
if err != nil {
return nil, nil, err
return res, err
}

var taskRegex *regexp.Regexp
taskRegex, err = alias.getTaskRegex()
if err != nil {
return nil, nil, err
return res, err
}

for _, variant := range p.BuildVariants {
Expand All @@ -1991,7 +1993,7 @@ func (p *Project) BuildProjectTVPairsWithAlias(aliases []ProjectAlias, requester
if bvtu.IsDisabled() || bvtu.SkipOnRequester(requester) {
continue
}
pairs = append(pairs, TVPair{variant.Name, t.Name})
res.ExecTasks = append(res.ExecTasks, TVPair{variant.Name, t.Name})
}
}

Expand All @@ -2002,12 +2004,12 @@ func (p *Project) BuildProjectTVPairsWithAlias(aliases []ProjectAlias, requester
if !taskRegex.MatchString(displayTask.Name) {
continue
}
displayTaskPairs = append(displayTaskPairs, TVPair{variant.Name, displayTask.Name})
res.DisplayTasks = append(res.DisplayTasks, TVPair{variant.Name, displayTask.Name})
}
}
}

return pairs, displayTaskPairs, nil
return res, nil
}

func (p *Project) VariantTasksForSelectors(definitions []patch.PatchTriggerDefinition, requester string) ([]patch.VariantTasks, error) {
Expand All @@ -2026,9 +2028,7 @@ func (p *Project) VariantTasksForSelectors(definitions []patch.PatchTriggerDefin
}
}

var err error
pairs := TaskVariantPairs{}
pairs.ExecTasks, pairs.DisplayTasks, err = p.BuildProjectTVPairsWithAlias(projectAliases, requester)
pairs, err := p.BuildProjectTVPairsWithAlias(projectAliases, requester)
if err != nil {
return nil, errors.Wrap(err, "getting pairs matching patch aliases")
}
Expand Down
85 changes: 39 additions & 46 deletions model/project_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -827,11 +827,11 @@ func (s *projectSuite) SetupTest() {

func (s *projectSuite) TestAliasResolution() {
// test that .* on variants and tasks selects everything
pairs, displayTaskPairs, err := s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[0]}, evergreen.PatchVersionRequester)
pairs, err := s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[0]}, evergreen.PatchVersionRequester)
s.NoError(err)
s.Len(pairs, 11)
pairStrs := make([]string, len(pairs))
for i, p := range pairs {
s.Len(pairs.ExecTasks, 11)
pairStrs := make([]string, len(pairs.ExecTasks))
for i, p := range pairs.ExecTasks {
pairStrs[i] = p.String()
}
s.Contains(pairStrs, "bv_1/a_task_1")
Expand All @@ -845,64 +845,64 @@ func (s *projectSuite) TestAliasResolution() {
s.Contains(pairStrs, "bv_2/b_task_1")
s.Contains(pairStrs, "bv_2/b_task_2")
s.Contains(pairStrs, "bv_2/another_disabled_task")
s.Require().Len(displayTaskPairs, 1)
s.Equal("bv_1/memes", displayTaskPairs[0].String())
s.Require().Len(pairs.DisplayTasks, 1)
s.Equal("bv_1/memes", pairs.DisplayTasks[0].String())

// test that the .*_2 regex on variants selects just bv_2
pairs, displayTaskPairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[1]}, evergreen.PatchVersionRequester)
pairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[1]}, evergreen.PatchVersionRequester)
s.NoError(err)
s.Len(pairs, 5)
for _, pair := range pairs {
s.Len(pairs.ExecTasks, 5)
for _, pair := range pairs.ExecTasks {
s.Equal("bv_2", pair.Variant)
}
s.Empty(displayTaskPairs)
s.Empty(pairs.DisplayTasks)

// test that the .*_2 regex on tasks selects just the _2 tasks
pairs, displayTaskPairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[2]}, evergreen.PatchVersionRequester)
pairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[2]}, evergreen.PatchVersionRequester)
s.NoError(err)
s.Len(pairs, 4)
for _, pair := range pairs {
s.Len(pairs.ExecTasks, 4)
for _, pair := range pairs.ExecTasks {
s.Contains(pair.TaskName, "task_2")
}
s.Empty(displayTaskPairs)
s.Empty(pairs.DisplayTasks)

// test that the 'a' tag only selects 'a' tasks
pairs, displayTaskPairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[3]}, evergreen.PatchVersionRequester)
pairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[3]}, evergreen.PatchVersionRequester)
s.NoError(err)
s.Len(pairs, 4)
for _, pair := range pairs {
s.Len(pairs.ExecTasks, 4)
for _, pair := range pairs.ExecTasks {
s.Contains(pair.TaskName, "a_task")
}
s.Empty(displayTaskPairs)
s.Empty(pairs.DisplayTasks)

// test that the .*_2 regex selects the union of both
pairs, displayTaskPairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[4]}, evergreen.PatchVersionRequester)
pairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[4]}, evergreen.PatchVersionRequester)
s.NoError(err)
s.Len(pairs, 4)
for _, pair := range pairs {
s.Len(pairs.ExecTasks, 4)
for _, pair := range pairs.ExecTasks {
s.NotEqual("b_task_1", pair.TaskName)
}
s.Empty(displayTaskPairs)
s.Empty(pairs.DisplayTasks)

// test for display tasks
pairs, displayTaskPairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[5]}, evergreen.PatchVersionRequester)
pairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[5]}, evergreen.PatchVersionRequester)
s.NoError(err)
s.Empty(pairs)
s.Require().Len(displayTaskPairs, 1)
s.Equal("bv_1/memes", displayTaskPairs[0].String())
s.Empty(pairs.ExecTasks)
s.Require().Len(pairs.DisplayTasks, 1)
s.Equal("bv_1/memes", pairs.DisplayTasks[0].String())

// test for alias including a task belong to a disabled variant
pairs, displayTaskPairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[6]}, evergreen.PatchVersionRequester)
pairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[6]}, evergreen.PatchVersionRequester)
s.NoError(err)
s.Empty(pairs)
s.Empty(displayTaskPairs)
s.Empty(pairs.ExecTasks)
s.Empty(pairs.DisplayTasks)

pairs, displayTaskPairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[8]}, evergreen.PatchVersionRequester)
pairs, err = s.project.BuildProjectTVPairsWithAlias([]ProjectAlias{s.aliases[8]}, evergreen.PatchVersionRequester)
s.NoError(err)
s.Require().Len(pairs, 2)
s.Equal("bv_2/a_task_1", pairs[0].String())
s.Equal("bv_2/a_task_2", pairs[1].String())
s.Empty(displayTaskPairs)
s.Require().Len(pairs.ExecTasks, 2)
s.Equal("bv_2/a_task_1", pairs.ExecTasks[0].String())
s.Equal("bv_2/a_task_2", pairs.ExecTasks[1].String())
s.Empty(pairs.DisplayTasks)
}

func (s *projectSuite) TestCheckRunCount() {
Expand Down Expand Up @@ -2838,23 +2838,16 @@ patch_aliases:
s.NotNil(pc)

alias := pc.PatchAliases[0]
pairs, _, err := p.BuildProjectTVPairsWithAlias([]ProjectAlias{alias}, evergreen.PatchVersionRequester)
pairs, err := p.BuildProjectTVPairsWithAlias([]ProjectAlias{alias}, evergreen.PatchVersionRequester)
s.NoError(err)
s.Len(pairs, 1)
for _, pair := range pairs {
a := pair.Variant
b := pair.TaskName
print(a, b)
}

pairStrs := make([]string, len(pairs))
for i, p := range pairs {
s.Len(pairs.ExecTasks, 1)
pairStrs := make([]string, len(pairs.ExecTasks))
for i, p := range pairs.ExecTasks {
pairStrs[i] = p.String()
}

s.Contains(pairStrs, "print-variant/print")

for _, pair := range pairs {
for _, pair := range pairs.ExecTasks {
s.NotEqual("performance-variant", pair.Variant)
s.NotEqual("other-variant", pair.Variant)
}
Expand Down
59 changes: 56 additions & 3 deletions validator/project_validator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ type projectConfigValidator func(config *model.ProjectConfig) ValidationErrors

type projectSettingsValidator func(context.Context, *evergreen.Settings, *model.Project, *model.ProjectRef, bool) ValidationErrors

type projectAliasValidator func(config *model.Project, aliases model.ProjectAliases) ValidationErrors

// bool indicates if we should still run the validator if the project is complex
type longValidator func(*model.Project, bool) ValidationErrors

Expand Down Expand Up @@ -166,6 +168,11 @@ var projectWarningValidators = []projectValidator{
checkTaskUsage,
}

var projectAliasWarningValidators = []projectAliasValidator{
validateAliasCoverage,
validateCheckRuns,
}

// Functions used to validate a project configuration that requires additional
// info such as admin settings and project settings.
var projectSettingsValidators = []projectSettingsValidator{
Expand Down Expand Up @@ -287,7 +294,7 @@ func CheckProject(ctx context.Context, project *model.Project, config *model.Pro
return append(verrs, CheckAliasWarnings(project, aliases)...)
}

// verify that the project configuration semantics is valid
// CheckProjectWarnings returns warnings about the project configuration semantics
func CheckProjectWarnings(project *model.Project) ValidationErrors {
validationErrs := ValidationErrors{}
for _, projectWarningValidator := range projectWarningValidators {
Expand All @@ -297,11 +304,18 @@ func CheckProjectWarnings(project *model.Project) ValidationErrors {
return validationErrs
}

// CheckAliasWarnings returns warnings related to the definition of tasks/variants matching the given aliases.
func CheckAliasWarnings(project *model.Project, aliases model.ProjectAliases) ValidationErrors {
return validateAliasCoverage(project, aliases)
validationErrs := ValidationErrors{}
for _, validator := range projectAliasWarningValidators {
validationErrs = append(validationErrs,
validator(project, aliases)...)
}

return validationErrs
}

// verify that the project configuration syntax is valid
// CheckProjectErrors returns errors about the project configuration syntax
func CheckProjectErrors(ctx context.Context, project *model.Project, includeLong bool) ValidationErrors {
validationErrs := ValidationErrors{}
for _, projectErrorValidator := range projectErrorValidators {
Expand Down Expand Up @@ -329,6 +343,7 @@ func CheckProjectErrors(ctx context.Context, project *model.Project, includeLong
return validationErrs
}

// CheckPatchedProjectConfigErrors returns validation errors for the given patched project config.
func CheckPatchedProjectConfigErrors(patchedProjectConfig string) ValidationErrors {
validationErrs := ValidationErrors{}
if len(patchedProjectConfig) <= 0 {
Expand Down Expand Up @@ -657,6 +672,44 @@ func getAliasCoverage(p *model.Project, aliasMap map[string]model.ProjectAlias)
return aliasNeedsVariant, aliasNeedsTask, nil
}

// validateCheckRuns returns warnings if PR aliases are going to violate check run rules for the given config.
func validateCheckRuns(p *model.Project, aliases model.ProjectAliases) ValidationErrors {
errs := ValidationErrors{}
aliasMap := map[string][]model.ProjectAlias{} // map of alias name to aliases
for _, a := range aliases {
if _, ok := aliasMap[a.Alias]; !ok {
aliasMap[a.Alias] = []model.ProjectAlias{}
}
aliasMap[a.Alias] = append(aliasMap[a.Alias], a)
}

tvPairs, err := p.BuildProjectTVPairsWithAlias(aliasMap[evergreen.GithubPRAlias], evergreen.GithubPRRequester)
if err != nil {
errs = append(errs, ValidationError{
Message: "problem getting task variant pairs for PR aliases",
Level: Warning,
})
return errs
}
tvPairs.ExecTasks, err = model.IncludeDependencies(p, tvPairs.ExecTasks, evergreen.GithubPRRequester, nil)
if err != nil {
errs = append(errs, ValidationError{
Message: "problem adding dependencies to PR alias tasks",
Level: Warning,
})
return errs
}
numCheckRuns := p.GetNumCheckRunsFromTaskVariantPairs(&tvPairs)
checkRunLimit := evergreen.GetEnvironment().Settings().GitHubCheckRun.CheckRunLimit
if numCheckRuns > checkRunLimit {
errs = append(errs, ValidationError{
Message: fmt.Sprintf("total number of checkRuns (%d) exceeds maximum limit (%d)", numCheckRuns, checkRunLimit),
Level: Warning,
})
}
return errs
}

func aliasMatchesTaskGroupTask(p *model.Project, alias model.ProjectAlias, tgName string) (bool, error) {
tg := p.FindTaskGroup(tgName)
if tg == nil {
Expand Down
Loading

0 comments on commit af843d2

Please sign in to comment.