DEVPROD-6045 Make generator/generated tasks track properly for bisect…

… stepback (#7709)

model/task/db.go

@@ -130,6 +130,9 @@ var (
 	LastPassingStepbackTaskIdKey = bsonutil.MustHaveTag(StepbackInfo{}, "LastPassingStepbackTaskId")
 	NextStepbackTaskIdKey        = bsonutil.MustHaveTag(StepbackInfo{}, "NextStepbackTaskId")
 	PreviousStepbackTaskIdKey    = bsonutil.MustHaveTag(StepbackInfo{}, "PreviousStepbackTaskId")
+	GeneratedStepbackInfoKey     = bsonutil.MustHaveTag(StepbackInfo{}, "GeneratedStepbackInfo")
+	StepbackInfoDisplayNameKey   = bsonutil.MustHaveTag(StepbackInfo{}, "DisplayName")
+	StepbackInfoBuildVariantKey  = bsonutil.MustHaveTag(StepbackInfo{}, "BuildVariant")
 )
 
 var (

model/task/task.go

@@ -327,8 +327,19 @@ type StepbackInfo struct {
 	NextStepbackTaskId string `bson:"next_stepback_task_id,omitempty" json:"next_stepback_task_id"`
 	// PreviousStepbackTaskId stores the last stepback iteration id.
 	PreviousStepbackTaskId string `bson:"previous_stepback_task_id,omitempty" json:"previous_stepback_task_id"`
+	// GeneratedStepbackInfo stores information on a generator for it's generated tasks.
+	GeneratedStepbackInfo []StepbackInfo `bson:"generated_stepback_info,omitempty" json:"generated_stepback_info,omitempty"`
+
+	// Generator fields only (responsible for propogating stepback in its generated tasks).
+	// DisplayName is the display name of the generated task.
+	DisplayName string `bson:"display_name,omitempty" json:"display_name,omitempty"`
+	// BuildVariant is the build variant of the generated task.
+	BuildVariant string `bson:"build_variant,omitempty" json:"build_variant,omitempty"`
 }
 
+// IsZero returns true if the StepbackInfo is empty or nil.
+// It does not include GeneratedStepbackInfo in the check because
+// those do not cause a generator to stepback.
 func (s *StepbackInfo) IsZero() bool {
 	if s == nil {
 		return true
@@ -340,6 +351,20 @@ func (s *StepbackInfo) IsZero() bool {
 	return true
 }
 
+// GetStepbackInfoForGeneratedTask returns the StepbackInfo for a generated task that's
+// on a generator task.
+func (s *StepbackInfo) GetStepbackInfoForGeneratedTask(displayName string, buildVariant string) *StepbackInfo {
+	if s == nil {
+		return nil
+	}
+	for _, info := range s.GeneratedStepbackInfo {
+		if info.DisplayName == displayName && info.BuildVariant == buildVariant {
+			return &info
+		}
+	}
+	return nil
+}
+
 // ExecutionPlatform indicates the type of environment that the task runs in.
 type ExecutionPlatform string
 
@@ -1370,7 +1395,7 @@ func findMidwayTask(t1, t2 Task) (*Task, error) {
 	catcher.ErrorfWhen(t1.BuildVariant != t2.BuildVariant, "given tasks have differing build variants '%s' and '%s'", t1.BuildVariant, t2.BuildVariant)
 	catcher.ErrorfWhen(t1.DisplayName != t2.DisplayName, "given tasks have differing display name '%s' and '%s'", t1.DisplayName, t2.DisplayName)
 	catcher.ErrorfWhen(t1.Project != t2.Project, "given tasks have differing project '%s' and '%s'", t1.Project, t2.Project)
-	catcher.ErrorfWhen(t1.Requester != t2.Requester, "given tasks have differing project '%s' and '%s'", t1.Requester, t2.Requester)
+	catcher.ErrorfWhen(t1.Requester != t2.Requester, "given tasks have differing requester '%s' and '%s'", t1.Requester, t2.Requester)
 	if catcher.HasErrors() {
 		return nil, catcher.Resolve()
 	}
@@ -1395,6 +1420,9 @@ func findMidwayTask(t1, t2 Task) (*Task, error) {
 // passed cannot have a middle (i.e. it is sequential tasks or the same task)
 // it will return the first task given.
 func FindMidwayTaskFromIds(t1Id, t2Id string) (*Task, error) {
+	if t1Id == "" || t2Id == "" {
+		return nil, nil
+	}
 	t1, err := FindOneId(t1Id)
 	if err != nil {
 		return nil, errors.Wrapf(err, "finding task id '%s'", t1Id)
@@ -1561,6 +1589,60 @@ func SetLastAndPreviousStepbackIds(taskId string, s StepbackInfo) error {
 	)
 }
 
+// AddGeneratedStepbackInfoForGenerator appends a new StepbackInfo to the
+// task's GeneratedStepbackInfo.
+func AddGeneratedStepbackInfoForGenerator(taskId string, s StepbackInfo) error {
+	return UpdateOne(
+		bson.M{
+			IdKey: taskId,
+		},
+		bson.M{
+			"$push": bson.M{
+				bsonutil.GetDottedKeyName(StepbackInfoKey, GeneratedStepbackInfoKey): s,
+			},
+		},
+	)
+}
+
+// SetGeneratedStepbackInfoForGenerator sets the StepbackInfo's GeneratedStepbackInfo
+// element with the same DisplayName and BuildVariant as the input StepbackInfo.
+func SetGeneratedStepbackInfoForGenerator(ctx context.Context, taskId string, s StepbackInfo) error {
+	r, err := evergreen.GetEnvironment().DB().Collection(Collection).UpdateOne(ctx,
+		bson.M{
+			IdKey: taskId,
+			StepbackInfoKey: bson.M{
+				GeneratedStepbackInfoKey: bson.M{
+					"$elemMatch": bson.M{
+						StepbackInfoDisplayNameKey:  s.DisplayName,
+						StepbackInfoBuildVariantKey: s.BuildVariant,
+					},
+				},
+			},
+		},
+		bson.M{
+			"$set": bson.M{
+				bsonutil.GetDottedKeyName(StepbackInfoKey, GeneratedStepbackInfoKey, "$[elem]", LastFailingStepbackTaskIdKey): s.LastFailingStepbackTaskId,
+				bsonutil.GetDottedKeyName(StepbackInfoKey, GeneratedStepbackInfoKey, "$[elem]", LastPassingStepbackTaskIdKey): s.LastPassingStepbackTaskId,
+				bsonutil.GetDottedKeyName(StepbackInfoKey, GeneratedStepbackInfoKey, "$[elem]", NextStepbackTaskIdKey):        s.NextStepbackTaskId,
+				bsonutil.GetDottedKeyName(StepbackInfoKey, GeneratedStepbackInfoKey, "$[elem]", PreviousStepbackTaskIdKey):    s.PreviousStepbackTaskId,
+			},
+		},
+		options.Update().SetArrayFilters(options.ArrayFilters{
+			Filters: []interface{}{
+				bson.M{
+					bsonutil.GetDottedKeyName("elem", DisplayNameKey):  s.DisplayName,
+					bsonutil.GetDottedKeyName("elem", BuildVariantKey): s.BuildVariant,
+				},
+			},
+		}),
+	)
+	// If no documents were modified, fallback to adding the new StepbackInfo.
+	if r.ModifiedCount == 0 {
+		return AddGeneratedStepbackInfoForGenerator(taskId, s)
+	}
+	return err
+}
+
 // SetNextStepbackId sets the NextStepbackTaskId for a given task id.
 func SetNextStepbackId(taskId string, s StepbackInfo) error {
 	return UpdateOne(

model/task_lifecycle.go

@@ -555,17 +555,6 @@ func doLinearStepback(ctx context.Context, t *task.Task) error {
 // doBisectStepback performs a bisect stepback on the task.
 // If there are no tasks to bisect, it no-ops.
 func doBisectStepback(ctx context.Context, t *task.Task) error {
-	// If we are a generated task, do stepback on our generator.
-	if t.GeneratedBy != "" {
-		generator, err := task.FindOneId(t.GeneratedBy)
-		if err != nil {
-			return errors.Wrapf(err, "finding generator '%s'", t.GeneratedBy)
-		}
-		if generator == nil {
-			return errors.Errorf("nil generator '%s'", t.GeneratedBy)
-		}
-		return doBisectStepback(ctx, generator)
-	}
 	// Do stepback for all execution tasks.
 	if t.DisplayOnly {
 		execTasks, err := task.Find(task.ByIds(t.ExecutionTasks))
@@ -577,7 +566,7 @@ func doBisectStepback(ctx context.Context, t *task.Task) error {
 			catcher.Add(doBisectStepback(ctx, &et))
 		}
 		if catcher.HasErrors() {
-			return catcher.Resolve()
+			return errors.Wrapf(catcher.Resolve(), "performing bisect stepback on display task '%s' execution tasks", t.Id)
 		}
 	}
 
@@ -671,6 +660,114 @@ func doBisectStepback(ctx context.Context, t *task.Task) error {
 	return nil
 }
 
+func doBisectStepbackForGeneratedTask(ctx context.Context, generator *task.Task, generated *task.Task) error {
+	var s task.StepbackInfo
+	if lastStepbackInfo := generator.StepbackInfo.GetStepbackInfoForGeneratedTask(generated.DisplayName, generated.BuildVariant); lastStepbackInfo != nil {
+		// Carry over from the last task.
+		s = *lastStepbackInfo
+	} else {
+		lastPassingGenerated, err := generated.PreviousCompletedTask(generated.Project, []string{evergreen.TaskSucceeded})
+		if err != nil {
+			return errors.Wrap(err, "locating previous successful task")
+		}
+		// Noop if there is no previous passing task.
+		if lastPassingGenerated == nil {
+			return nil
+		}
+		if lastPassingGenerated.GeneratedBy == "" {
+			return errors.Errorf("last passing generated task '%s' must have a generator task", lastPassingGenerated.Id)
+		}
+		s = task.StepbackInfo{
+			DisplayName:               generated.DisplayName,
+			BuildVariant:              generated.BuildVariant,
+			LastPassingStepbackTaskId: lastPassingGenerated.GeneratedBy,
+		}
+		grip.Info(message.Fields{
+			"message":                       "starting bisect stepback on generator task",
+			"last_passing_stepback_task_id": s.LastPassingStepbackTaskId,
+			"generator_task_id":             generator.Id,
+			"generated_task_id":             generated.Id,
+			"display_name":                  generated.DisplayName,
+			"BuildVariant":                  generated.BuildVariant,
+			"gap":                           generated.RevisionOrderNumber - lastPassingGenerated.RevisionOrderNumber,
+			"project_id":                    generated.Project,
+		})
+	}
+
+	// The previous iteration is the task we are currently processing.
+	s.PreviousStepbackTaskId = generator.Id
+
+	// Depending on the task status, we want to update the
+	// last failing or last passing task.
+	if generated.Status == evergreen.TaskSucceeded {
+		s.LastPassingStepbackTaskId = generator.Id
+	} else if generated.Status == evergreen.TaskFailed {
+		s.LastFailingStepbackTaskId = generator.Id
+	} else {
+		return errors.Errorf("stopping task stepback due to status '%s'", generated.Status)
+	}
+
+	// The midway task is our next stepback target.
+	nextTask, err := task.FindMidwayTaskFromIds(s.LastFailingStepbackTaskId, s.LastPassingStepbackTaskId)
+	if err != nil {
+		return errors.Wrapf(err, "finding midway task between tasks '%s' and '%s'", s.LastFailingStepbackTaskId, s.LastPassingStepbackTaskId)
+	}
+	if nextTask == nil {
+		return errors.Errorf("midway task could not be found for tasks '%s' '%s'", s.LastFailingStepbackTaskId, s.LastPassingStepbackTaskId)
+	}
+	// If our next task is last passing Id, we have finished stepback.
+	if nextTask.Id == s.LastPassingStepbackTaskId {
+		return nil
+	}
+	s.NextStepbackTaskId = nextTask.Id
+	// Store our next task to our current task.
+	if err := task.SetGeneratedStepbackInfoForGenerator(ctx, generator.Id, s); err != nil {
+		return errors.Wrapf(err, "could not set generated stepback task id for stepback task '%s'", generator.Id)
+	}
+	// This is only for UI purposes. The generated task needs these fields populated
+	// to show stepback options in the UI. We create a new stepback info because we do
+	// not want to copy over the generated related fields.
+	if err := task.SetLastAndPreviousStepbackIds(generated.Id, task.StepbackInfo{
+		LastFailingStepbackTaskId: s.LastFailingStepbackTaskId,
+		LastPassingStepbackTaskId: s.LastPassingStepbackTaskId,
+		PreviousStepbackTaskId:    s.PreviousStepbackTaskId,
+		NextStepbackTaskId:        s.NextStepbackTaskId,
+	}); err != nil {
+		return errors.Wrapf(err, "could not set stepback info for generated task '%s'", generated.Id)
+	}
+	// Store our last and previous stepback tasks in our upcoming/next task.
+	if err = task.AddGeneratedStepbackInfoForGenerator(nextTask.Id, s); err != nil {
+		return errors.Wrapf(err, "setting stepback info for task '%s'", nextTask.Id)
+	}
+
+	grip.Info(message.Fields{
+		"message":                         "bisect stepback on generator task",
+		"last_failing_stepback_task_id":   s.LastFailingStepbackTaskId,
+		"last_passing_stepback_task_id":   s.LastPassingStepbackTaskId,
+		"next_task_id":                    nextTask.Id,
+		"next_task_display_name":          nextTask.DisplayName,
+		"next_task_build_id":              nextTask.BuildId,
+		"last_stepback_task_generator_id": generator.Id,
+		"last_stepback_task_id":           generated.Id,
+		"last_stepback_task_status":       generated.Status,
+		"project_id":                      generated.Project,
+	})
+
+	// Activate the next task.
+	if err = SetActiveState(ctx, evergreen.StepbackTaskActivator, true, *nextTask); err != nil {
+		return errors.Wrapf(err, "setting task '%s' active", nextTask.Id)
+	}
+
+	// If this is a generator task, activate generated tasks.
+	if nextTask.GenerateTask {
+		if err = nextTask.SetGeneratedTasksToActivate(generated.BuildVariant, generated.DisplayName); err != nil {
+			return errors.Wrap(err, "setting generated tasks to activate")
+		}
+	}
+
+	return nil
+}
+
 // MarkEnd updates the task as being finished, performs a stepback if necessary, and updates the build status
 func MarkEnd(ctx context.Context, settings *evergreen.Settings, t *task.Task, caller string, finishTime time.Time, detail *apimodels.TaskEndDetail,
 	deactivatePrevious bool) error {
@@ -780,9 +877,9 @@ func MarkEnd(ctx context.Context, settings *evergreen.Settings, t *task.Task, ca
 			if err != nil {
 				return errors.Wrap(err, "getting display task")
 			}
-			err = evalStepback(ctx, t.DisplayTask, caller, status, deactivatePrevious)
+			err = evalStepback(ctx, t.DisplayTask, status)
 		} else {
-			err = evalStepback(ctx, t, caller, status, deactivatePrevious)
+			err = evalStepback(ctx, t, status)
 		}
 		grip.Error(message.WrapError(err, message.Fields{
 			"message": "evaluating stepback",
@@ -791,6 +888,13 @@ func MarkEnd(ctx context.Context, settings *evergreen.Settings, t *task.Task, ca
 		}))
 	}
 
+	// Deactivate previous occurrences of the same task if this one passed on mainline commits.
+	if t.Status == evergreen.TaskSucceeded && deactivatePrevious && t.Requester == evergreen.RepotrackerVersionRequester && t.ActivatedBy != evergreen.StepbackTaskActivator {
+		grip.Error(message.WrapError(DeactivatePreviousTasks(ctx, t, caller), message.Fields{
+			"message": "deactivating previous tasks",
+		}))
+	}
+
 	if err = UpdateBuildAndVersionStatusForTask(ctx, t); err != nil {
 		return errors.Wrap(err, "updating build/version status")
 	}
@@ -1254,7 +1358,7 @@ func removeNextMergeTaskDependency(ctx context.Context, cq commitqueue.CommitQue
 // evalStepback runs linear or bisect stepback depending on project, build variant, and task settings.
 // The status passed in is a display status, initially stepback only activates if the task
 // has failed but not on system failure.
-func evalStepback(ctx context.Context, t *task.Task, caller, status string, deactivatePrevious bool) error {
+func evalStepback(ctx context.Context, t *task.Task, status string) error {
 	s, err := getStepback(t.Id)
 	if err != nil {
 		return errors.WithStack(err)
@@ -1263,14 +1367,14 @@ func evalStepback(ctx context.Context, t *task.Task, caller, status string, deac
 	// and it is not aborted.
 	newStepback := status == evergreen.TaskFailed && !t.Aborted
 	if s.bisect {
-		return evalBisectStepback(ctx, t, caller, newStepback, s.shouldStepback)
+		return evalBisectStepback(ctx, t, newStepback, s.shouldStepback)
 	}
-	return evalLinearStepback(ctx, t, caller, newStepback, s.shouldStepback, deactivatePrevious)
+	return evalLinearStepback(ctx, t, newStepback, s.shouldStepback)
 }
 
 // evalLinearStepback performs linear stepback on the task or cleans up after previous iterations of lienar
 // stepback.
-func evalLinearStepback(ctx context.Context, t *task.Task, caller string, newStepback, shouldStepback, deactivatePrevious bool) error {
+func evalLinearStepback(ctx context.Context, t *task.Task, newStepback, shouldStepback bool) error {
 	existingStepback := t.Status == evergreen.TaskFailed && t.ActivatedBy == evergreen.StepbackTaskActivator
 	if newStepback || existingStepback {
 		if !shouldStepback {
@@ -1297,27 +1401,39 @@ func evalLinearStepback(ctx context.Context, t *task.Task, caller string, newSte
 			return catcher.Resolve()
 		}
 		return errors.Wrap(doLinearStepback(ctx, t), "performing linear stepback")
-	} else if t.Status == evergreen.TaskSucceeded && deactivatePrevious && t.Requester == evergreen.RepotrackerVersionRequester {
-		// When stepback finishes (the task is successful), and stepback was done on mainline commits,
-		// ignore running previous activated tasks for this build variant.
-		return errors.Wrap(DeactivatePreviousTasks(ctx, t, caller), "deactivating previous tasks")
 	}
 	return nil
 }
 
 // evalBisectStepback performs bisect stepback on the task.
-func evalBisectStepback(ctx context.Context, t *task.Task, caller string, newStepback, shouldStepback bool) error {
+func evalBisectStepback(ctx context.Context, t *task.Task, newStepback, shouldStepback bool) error {
 	// If the task is aborted or stepback is disabled then no-op.
 	if t.Aborted || !shouldStepback {
 		return nil
 	}
 
 	existingStepback := !t.StepbackInfo.IsZero() && t.ActivatedBy == evergreen.StepbackTaskActivator
-	if newStepback || existingStepback {
+	if (newStepback || existingStepback) && t.GeneratedBy == "" {
 		return errors.Wrap(doBisectStepback(ctx, t), "performing bisect stepback")
 	}
 
-	return nil
+	if t.GeneratedBy == "" {
+		return nil
+	}
+
+	generator, err := task.FindOneId(t.GeneratedBy)
+	if err != nil {
+		return errors.Wrapf(err, "finding generator '%s'", t.GeneratedBy)
+	}
+	if generator == nil {
+		return errors.Errorf("nil generator '%s'", t.GeneratedBy)
+	}
+	// Check if the task is in stepback (if it is not a new one).
+	if generator.StepbackInfo.GetStepbackInfoForGeneratedTask(t.DisplayName, t.BuildVariant) == nil && !newStepback {
+		return nil
+	}
+
+	return errors.Wrapf(doBisectStepbackForGeneratedTask(ctx, generator, t), "bisect stepback on generator '%s' for generated '%s'", t.GeneratedBy, t.Id)
 }
 
 // updateMakespans updates the predicted and actual makespans for the tasks in