-
Notifications
You must be signed in to change notification settings - Fork 476
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat:Poll Images from ECR Container Repository Plugin #3971
Changes from 41 commits
c4f779a
3b1ffbc
52c9fbd
093a62c
90488cc
89e3756
e702b8c
51bd92b
3955d8a
4b5a07a
b4715c7
027e14c
91a2301
4ec2bf7
65987f6
5500856
1641402
3f10998
affdcb0
5f776dc
e50001d
3196231
6f1a638
937dd39
4450ec4
77dccf4
6566e3b
898e850
8157055
d67f4a9
f5b3754
d35a960
5045b11
71244b4
0ef3cba
4cc8917
6d697ad
0e01681
db10a7b
14a8b60
f1594f2
8bf9892
542df85
184a1e5
9f3d2fc
539d824
f957c9d
027ff85
338f2ba
6169e24
294b535
ce303aa
84ffe61
bd8a516
c525146
faac3e1
a47fe17
082bac2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,119 @@ | ||
package cron | ||
|
||
import ( | ||
"fmt" | ||
"github.com/caarlos0/env" | ||
repository2 "github.com/devtron-labs/devtron/internal/sql/repository" | ||
"github.com/devtron-labs/devtron/pkg/bean" | ||
"github.com/devtron-labs/devtron/pkg/pipeline" | ||
bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" | ||
"github.com/devtron-labs/devtron/pkg/pipeline/repository" | ||
repository3 "github.com/devtron-labs/devtron/pkg/plugin/repository" | ||
"github.com/robfig/cron/v3" | ||
"go.uber.org/zap" | ||
) | ||
|
||
type CiTriggerCron interface { | ||
TriggerCiCron() | ||
} | ||
|
||
type CiTriggerCronImpl struct { | ||
logger *zap.SugaredLogger | ||
cron *cron.Cron | ||
cfg *CiTriggerCronConfig | ||
pipelineStageRepository repository.PipelineStageRepository | ||
ciHandler pipeline.CiHandler | ||
ciArtifactRepository repository2.CiArtifactRepository | ||
globalPluginRepository repository3.GlobalPluginRepository | ||
} | ||
|
||
func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, pipelineStageRepository repository.PipelineStageRepository, | ||
ciHandler pipeline.CiHandler, ciArtifactRepository repository2.CiArtifactRepository, globalPluginRepository repository3.GlobalPluginRepository) *CiTriggerCronImpl { | ||
cron := cron.New( | ||
cron.WithChain()) | ||
cron.Start() | ||
impl := &CiTriggerCronImpl{ | ||
logger: logger, | ||
cron: cron, | ||
pipelineStageRepository: pipelineStageRepository, | ||
ciHandler: ciHandler, | ||
cfg: cfg, | ||
ciArtifactRepository: ciArtifactRepository, | ||
globalPluginRepository: globalPluginRepository, | ||
} | ||
|
||
_, err := cron.AddFunc(fmt.Sprintf("@every %dm", cfg.SourceControllerCronTime), impl.TriggerCiCron) | ||
if err != nil { | ||
logger.Errorw("error while configure cron job for ci workflow status update", "err", err) | ||
return impl | ||
} | ||
return impl | ||
} | ||
|
||
type CiTriggerCronConfig struct { | ||
SourceControllerCronTime int `env:"CI_TRIGGER_CRON_TIME" envDefault:"2"` | ||
PluginName string `env:"PLUGIN_IDS" envDefault:"Pull images from container repository"` | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. PluginName or PluginIds |
||
} | ||
|
||
func GetCiTriggerCronConfig() (*CiTriggerCronConfig, error) { | ||
cfg := &CiTriggerCronConfig{} | ||
err := env.Parse(cfg) | ||
if err != nil { | ||
fmt.Println("failed to parse ci trigger cron config: " + err.Error()) | ||
return nil, err | ||
} | ||
return cfg, nil | ||
} | ||
|
||
// UpdateCiWorkflowStatusFailedCron this function will execute periodically | ||
func (impl *CiTriggerCronImpl) TriggerCiCron() { | ||
|
||
plugin, err := impl.globalPluginRepository.GetPluginByName(impl.cfg.PluginName) | ||
|
||
if err != nil || len(plugin) == 0 { | ||
return | ||
} | ||
|
||
ciPipelineIds, err := impl.pipelineStageRepository.GetAllCiPipelineIdsByPluginIdAndStageType(plugin[0].Id, string(repository.PIPELINE_STAGE_TYPE_PRE_CI)) | ||
if err != nil { | ||
return | ||
} | ||
//artifacts, err := impl.ciArtifactRepository.GetLatestArtifactTimeByCiPipelineIds(ciPipelineIds) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. remove unwanted comments |
||
//mp := make(map[int]time.Time) | ||
//for _, artifact := range artifacts { | ||
// mp[artifact.PipelineId] = artifact.CreatedOn | ||
//} | ||
for _, ciPipelineId := range ciPipelineIds { | ||
//_, err := impl.ciHandler.FetchMaterialsByPipelineId(ciPipelineId, false) | ||
//if err != nil { | ||
// return | ||
//} | ||
var ciPipelineMaterials []bean.CiPipelineMaterial | ||
|
||
//for _, material := range materials { | ||
// if len(material.History) == 0 { | ||
// return | ||
// } | ||
// ciPipelineMaterial := bean.CiPipelineMaterial{ | ||
// Id: material.Id, | ||
// GitMaterialId: material.GitMaterialId, | ||
// GitCommit: bean.GitCommit{ | ||
// Commit: material.History[0].Commit, | ||
// }, | ||
// } | ||
// ciPipelineMaterials = append(ciPipelineMaterials, ciPipelineMaterial) | ||
//} | ||
ciTriggerRequest := bean.CiTriggerRequest{ | ||
PipelineId: ciPipelineId, | ||
CiPipelineMaterial: ciPipelineMaterials, | ||
TriggeredBy: 1, | ||
InvalidateCache: false, | ||
PipelineType: bean2.CI_JOB, | ||
} | ||
_, err = impl.ciHandler.HandleCIManual(ciTriggerRequest) | ||
if err != nil { | ||
return | ||
} | ||
} | ||
return | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -20,6 +20,7 @@ package repository | |
import ( | ||
"encoding/json" | ||
"fmt" | ||
"github.com/devtron-labs/devtron/internal/sql/repository/helper" | ||
"github.com/devtron-labs/devtron/pkg/sql" | ||
"strings" | ||
"time" | ||
|
@@ -59,6 +60,8 @@ type CiArtifactRepository interface { | |
GetByWfId(wfId int) (artifact *CiArtifact, err error) | ||
GetArtifactsByCDPipeline(cdPipelineId, limit int, parentId int, parentType bean.WorkflowType) ([]*CiArtifact, error) | ||
|
||
GetLatestArtifactTimeByCiPipelineIds(ciPipelineIds []int) ([]*CiArtifact, error) | ||
GetLatestArtifactTimeByCiPipelineId(ciPipelineId int) (*CiArtifact, error) | ||
GetArtifactsByCDPipelineV2(cdPipelineId int) ([]CiArtifact, error) | ||
GetArtifactsByCDPipelineAndRunnerType(cdPipelineId int, runnerType bean.WorkflowType) ([]CiArtifact, error) | ||
SaveAll(artifacts []*CiArtifact) error | ||
|
@@ -236,6 +239,36 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipeline(cdPipelineId, limi | |
return artifactsAll, err | ||
} | ||
|
||
func (impl CiArtifactRepositoryImpl) GetLatestArtifactTimeByCiPipelineIds(ciPipelineIds []int) ([]*CiArtifact, error) { | ||
artifacts := make([]*CiArtifact, 0) | ||
query := "select cws.pipeline_id, cws.created_on from " + | ||
"(SELECT pipeline_id, MAX(created_on) created_on " + | ||
"FROM ci_artifact " + | ||
"GROUP BY pipeline_id) cws " + | ||
"where cws.pipeline_id IN (" + helper.GetCommaSepratedString(ciPipelineIds) + "); " | ||
|
||
_, err := impl.dbConnection.Query(&artifacts, query) | ||
if err != nil { | ||
return nil, err | ||
} | ||
return artifacts, nil | ||
} | ||
|
||
func (impl CiArtifactRepositoryImpl) GetLatestArtifactTimeByCiPipelineId(ciPipelineId int) (*CiArtifact, error) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. put comments |
||
artifacts := &CiArtifact{} | ||
query := "select cws.pipeline_id, cws.created_on from " + | ||
"(SELECT pipeline_id, MAX(created_on) created_on " + | ||
"FROM ci_artifact " + | ||
"GROUP BY pipeline_id) cws " + | ||
"where cws.pipeline_id = ? ; " | ||
|
||
_, err := impl.dbConnection.Query(artifacts, query, ciPipelineId) | ||
if err != nil { | ||
return nil, err | ||
} | ||
return artifacts, nil | ||
} | ||
|
||
func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipelineAndRunnerType(cdPipelineId int, runnerType bean.WorkflowType) ([]CiArtifact, error) { | ||
var artifactsA []CiArtifact | ||
var artifactsAB []CiArtifact | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
bean1.SKIP_BUILD_BUILD_TYPE >> bean1.SKIP_BUILD_TYPE