From 1fc038b5effa570e3bd6d6932102d8a1f807a058 Mon Sep 17 00:00:00 2001 From: adi6859 Date: Thu, 1 Feb 2024 17:09:16 +0530 Subject: [PATCH 01/45] story(configDiffView) : open api spec --- specs/configDiffView.yaml | 67 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 specs/configDiffView.yaml diff --git a/specs/configDiffView.yaml b/specs/configDiffView.yaml new file mode 100644 index 00000000000..7834e0b6ce1 --- /dev/null +++ b/specs/configDiffView.yaml @@ -0,0 +1,67 @@ +openapi: 3.0.0 +info: + title: Orchestrator Config Autocomplete API + version: 1.0.0 +paths: + /orchestrator/config/autocomplete: + get: + summary: Retrieve autocomplete data for configuration based on the provided appId and envId. The response includes configuration definitions with names, draft states, and types. + parameters: + - name: appId + in: query + description: The application ID. + required: true + schema: + type: string + - name: envId + in: query + description: The environment ID. + required: true + schema: + type: string + responses: + '200': + description: Successful response + content: + application/json: + example: + resourceConfig: + - name: "SampleConfig" + configState: 1 + type: "ConfigMap" + - name: "SecretConfig" + configState: 2 + type: "Secret" + - name: "DeploymentConfig" + configState: 3 + type: "Deployment Template" + + '400': + description: Bad Request + content: + application/json: + example: + #In case of Invalid appId and envId + error: "Invalid input parameters" + + '404': + description: Not Found + content: + application/json: + example: + #when BaseDeployment Template is not configured + error: "Resource not found" + '500': + description: Internal Server Error + content: + application/json: + example: + error: "Internal Server Error" + '403': + description: Forbidden + content: + application/json: + example: + #when not have access of particular resources + error: "Forbidden" + From 8eda3681199d60cf3d7c7dcc9b03e21eb503a10f Mon Sep 17 00:00:00 2001 From: adi6859 Date: Thu, 1 Feb 2024 18:43:09 +0530 Subject: [PATCH 02/45] story(configDiffView) : open api spec updated --- specs/configDiffView.yaml | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/specs/configDiffView.yaml b/specs/configDiffView.yaml index 7834e0b6ce1..b0d1ee6f4df 100644 --- a/specs/configDiffView.yaml +++ b/specs/configDiffView.yaml @@ -24,17 +24,10 @@ paths: description: Successful response content: application/json: - example: - resourceConfig: - - name: "SampleConfig" - configState: 1 - type: "ConfigMap" - - name: "SecretConfig" - configState: 2 - type: "Secret" - - name: "DeploymentConfig" - configState: 3 - type: "Deployment Template" + schema: + type: array + items: + $ref: "#/components/schemas/ResourceConfig" '400': description: Bad Request @@ -65,3 +58,23 @@ paths: #when not have access of particular resources error: "Forbidden" +components: + schemas: + ResourceConfig: + type: object + properties: + name: + type: string + description: Name of config + example: cm-1 + nullable: true + configState: + type: integer + description: state of config{1 represent published state ,2 represents draft state , 3 represent approval pending state } + example: 1 + nullable: false + type: + type: string + description: describe the config type {possible values are ConfigMap, Secret, Deployment Template} + example: ConfigMap + nullable: true \ No newline at end of file From 2d00c6a1e0a52144b81bc01f5026729f70f1856d Mon Sep 17 00:00:00 2001 From: adi6859 Date: Thu, 1 Feb 2024 18:46:29 +0530 Subject: [PATCH 03/45] story(configDiffView) : open api spec updated for error state --- specs/configDiffView.yaml | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/specs/configDiffView.yaml b/specs/configDiffView.yaml index b0d1ee6f4df..4f44994fcae 100644 --- a/specs/configDiffView.yaml +++ b/specs/configDiffView.yaml @@ -29,34 +29,14 @@ paths: items: $ref: "#/components/schemas/ResourceConfig" - '400': - description: Bad Request - content: - application/json: - example: - #In case of Invalid appId and envId - error: "Invalid input parameters" - - '404': - description: Not Found - content: - application/json: - example: - #when BaseDeployment Template is not configured - error: "Resource not found" '500': - description: Internal Server Error - content: - application/json: - example: - error: "Internal Server Error" + description: will get this response if any failure occurs at server side. + '400': + description: will get this response if invalid payload is sent in the request. '403': - description: Forbidden - content: - application/json: - example: - #when not have access of particular resources - error: "Forbidden" + description: will get this response if user doesn't view access permission for the app or env + '404': + description: will get this when BaseDeployment Template is not configured components: schemas: From 00d1e6cc36050271f2ddea65e830074d42439c84 Mon Sep 17 00:00:00 2001 From: adi6859 Date: Thu, 1 Feb 2024 22:10:43 +0530 Subject: [PATCH 04/45] story(configDiffView) : WIP --- Wire.go | 8 ++ .../DeploymentConfigurationRestHandler.go | 82 +++++++++++++++++++ api/router/DeploymentConfigRouter.go | 26 ++++++ api/router/router.go | 11 ++- .../chartConfig/ConfigMapRepository.go | 25 ++++++ pkg/config/DeploymentConfigurationService.go | 46 +++++++++++ pkg/config/bean.go | 24 ++++++ wire_gen.go | 13 ++- 8 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 api/restHandler/DeploymentConfigurationRestHandler.go create mode 100644 api/router/DeploymentConfigRouter.go create mode 100644 pkg/config/DeploymentConfigurationService.go create mode 100644 pkg/config/bean.go diff --git a/Wire.go b/Wire.go index 2d58868a603..5bd85879f27 100644 --- a/Wire.go +++ b/Wire.go @@ -99,6 +99,7 @@ import ( "github.com/devtron-labs/devtron/pkg/chart" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/commonService" + "github.com/devtron-labs/devtron/pkg/config" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" @@ -671,6 +672,13 @@ func InitializeApp() (*App, error) { scopedVariable.NewScopedVariableRestHandlerImpl, wire.Bind(new(scopedVariable.ScopedVariableRestHandler), new(*scopedVariable.ScopedVariableRestHandlerImpl)), + router.NewDeploymentConfigurationRouter, + wire.Bind(new(router.DeploymentConfigurationRouter), new(*router.DeploymentConfigurationRouterImpl)), + restHandler.NewDeploymentConfigurationRestHandlerImpl, + wire.Bind(new(restHandler.DeploymentConfigurationRestHandler), new(*restHandler.DeploymentConfigurationRestHandlerImpl)), + config.NewDeploymentConfigurationServiceImpl, + wire.Bind(new(config.DeploymentConfigurationService), new(*config.DeploymentConfigurationServiceImpl)), + util.NewGitCliUtil, router.NewTelemetryRouterImpl, diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go new file mode 100644 index 00000000000..09fb33feeaf --- /dev/null +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -0,0 +1,82 @@ +package restHandler + +import ( + "fmt" + "github.com/devtron-labs/devtron/api/restHandler/common" + "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" + "github.com/devtron-labs/devtron/pkg/auth/user" + "github.com/devtron-labs/devtron/pkg/config" + "github.com/devtron-labs/devtron/util/rbac" + "github.com/gorilla/mux" + "go.uber.org/zap" + "gopkg.in/go-playground/validator.v9" + "net/http" + "strconv" +) + +type DeploymentConfigurationRestHandler interface { + ConfigAutoComplete(w http.ResponseWriter, r *http.Request) +} +type DeploymentConfigurationRestHandlerImpl struct { + logger *zap.SugaredLogger + userAuthService user.UserService + validator *validator.Validate + enforcerUtil rbac.EnforcerUtil + enforcer casbin.Enforcer + deploymentConfigurationService config.DeploymentConfigurationService +} + +func NewDeploymentConfigurationRestHandlerImpl(logger *zap.SugaredLogger, + userAuthService user.UserService, + enforcerUtil rbac.EnforcerUtil, + enforcer casbin.Enforcer, + deploymentConfigurationService config.DeploymentConfigurationService, +) *DeploymentConfigurationRestHandlerImpl { + return &DeploymentConfigurationRestHandlerImpl{ + logger: logger, + userAuthService: userAuthService, + enforcerUtil: enforcerUtil, + enforcer: enforcer, + deploymentConfigurationService: deploymentConfigurationService, + } +} + +func (handler *DeploymentConfigurationRestHandlerImpl) ConfigAutoComplete(w http.ResponseWriter, r *http.Request) { + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + vars := mux.Vars(r) + //todo aditya get from query params + appId, err := strconv.Atoi(vars["appId"]) + if err != nil { + handler.logger.Errorw("request err, CSEnvironmentFetch", "err", err, "appId", appId) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + envId, err := strconv.Atoi(vars["envId"]) + if err != nil { + handler.logger.Errorw("bad request", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + + //RBAC START + token := r.Header.Get("token") + object := handler.enforcerUtil.GetAppRBACNameByAppId(appId) + ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, casbin.ActionGet) + if !ok { + common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), nil, http.StatusForbidden) + return + } + //RBAC END + + res, err := handler.deploymentConfigurationService.ConfigAutoComplete(appId, envId) + if err != nil { + handler.logger.Errorw("service err, CSEnvironmentFetch", "err", err, "appId", appId) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, err, res, http.StatusOK) +} diff --git a/api/router/DeploymentConfigRouter.go b/api/router/DeploymentConfigRouter.go new file mode 100644 index 00000000000..b79409f9c51 --- /dev/null +++ b/api/router/DeploymentConfigRouter.go @@ -0,0 +1,26 @@ +package router + +import ( + "github.com/devtron-labs/devtron/api/restHandler" + "github.com/gorilla/mux" +) + +type DeploymentConfigurationRouter interface { + initDeploymentConfigurationRouter(configRouter *mux.Router) +} + +type DeploymentConfigurationRouterImpl struct { + deploymentGroupRestHandler restHandler.DeploymentConfigurationRestHandler +} + +func NewDeploymentConfigurationRouter(deploymentGroupRestHandler restHandler.DeploymentConfigurationRestHandler) *DeploymentConfigurationRouterImpl { + router := &DeploymentConfigurationRouterImpl{ + deploymentGroupRestHandler: deploymentGroupRestHandler, + } + return router +} + +func (router DeploymentConfigurationRouterImpl) initDeploymentConfigurationRouter(configRouter *mux.Router) { + configRouter.Path("/autocomplete"). + HandlerFunc(router.deploymentGroupRestHandler.ConfigAutoComplete).Methods("POST") +} diff --git a/api/router/router.go b/api/router/router.go index 58375546cee..8437aa4a720 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -122,6 +122,7 @@ type MuxRouter struct { rbacRoleRouter user.RbacRoleRouter scopedVariableRouter ScopedVariableRouter ciTriggerCron cron.CiTriggerCron + deploymentConfigurationRouter DeploymentConfigurationRouter } func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, PipelineConfigRouter PipelineConfigRouter, @@ -154,7 +155,9 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P rbacRoleRouter user.RbacRoleRouter, scopedVariableRouter ScopedVariableRouter, ciTriggerCron cron.CiTriggerCron, - proxyRouter proxy.ProxyRouter) *MuxRouter { + proxyRouter proxy.ProxyRouter, + deploymentConfigurationRouter DeploymentConfigurationRouter, +) *MuxRouter { r := &MuxRouter{ Router: mux.NewRouter(), HelmRouter: HelmRouter, @@ -224,6 +227,7 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P rbacRoleRouter: rbacRoleRouter, scopedVariableRouter: scopedVariableRouter, ciTriggerCron: ciTriggerCron, + deploymentConfigurationRouter: deploymentConfigurationRouter, } return r } @@ -309,8 +313,9 @@ func (r MuxRouter) Init() { chartRefRouter := r.Router.PathPrefix("/orchestrator/chartref").Subrouter() r.ChartRefRouter.initChartRefRouter(chartRefRouter) - configMapRouter := r.Router.PathPrefix("/orchestrator/config").Subrouter() - r.ConfigMapRouter.initConfigMapRouter(configMapRouter) + configRouter := r.Router.PathPrefix("/orchestrator/config/").Subrouter() + r.ConfigMapRouter.initConfigMapRouter(configRouter) + r.deploymentConfigurationRouter.initDeploymentConfigurationRouter(configRouter) appStoreRouter := r.Router.PathPrefix("/orchestrator/app-store").Subrouter() r.AppStoreRouter.Init(appStoreRouter) diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index 69cd68013c1..008f3811a68 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -39,6 +39,8 @@ type ConfigMapRepository interface { GetByAppIdAppLevel(appId int) (*ConfigMapAppModel, error) GetByAppIdAndEnvIdEnvLevel(appId int, envId int) (*ConfigMapEnvModel, error) GetEnvLevelByAppId(appId int) ([]*ConfigMapEnvModel, error) + GetConfigNamesAppLevel(appId int) ([]CMCSNames, error) + GetConfigNamesEnvLevel(appId int, envId int) ([]CMCSNames, error) } type ConfigMapRepositoryImpl struct { @@ -58,6 +60,29 @@ type ConfigMapAppModel struct { SecretData string `sql:"secret_data"` sql.AuditLog } +type CMCSNames struct { + CMName string `json:"cm_name"` + CSName string `json:"cs_name"` +} + +func (impl ConfigMapRepositoryImpl) GetConfigNamesAppLevel(appId int) ([]CMCSNames, error) { + var cMCSNames []CMCSNames + query := "SELECT json_array_elements(config_map_data::json->'maps')->>'name' AS cm_name, " + + "json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name " + + "FROM config_map_app_level WHERE app_id = ?;" + + _, err := impl.dbConnection.Query(cMCSNames, query, appId) + return cMCSNames, err +} +func (impl ConfigMapRepositoryImpl) GetConfigNamesEnvLevel(appId int, envId int) ([]CMCSNames, error) { + var cMCSNames []CMCSNames + query := "SELECT json_array_elements(config_map_data::json->'maps')->>'name' AS cm_name, " + + "json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name " + + "FROM config_map_app_level WHERE app_id = ? AND env_id=?;" + + _, err := impl.dbConnection.Query(cMCSNames, query, appId, envId) + return cMCSNames, err +} func (impl ConfigMapRepositoryImpl) CreateAppLevel(model *ConfigMapAppModel) (*ConfigMapAppModel, error) { err := impl.dbConnection.Insert(model) diff --git a/pkg/config/DeploymentConfigurationService.go b/pkg/config/DeploymentConfigurationService.go new file mode 100644 index 00000000000..89f7d0dc7c4 --- /dev/null +++ b/pkg/config/DeploymentConfigurationService.go @@ -0,0 +1,46 @@ +package config + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "go.uber.org/zap" +) + +type DeploymentConfigurationService interface { + ConfigAutoComplete(appId int, envId int) (*ConfigDataResponse, error) +} + +type DeploymentConfigurationServiceImpl struct { + logger *zap.SugaredLogger + configMapRepository chartConfig.ConfigMapRepository +} + +func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, configMapRepository chartConfig.ConfigMapRepository) (*DeploymentConfigurationServiceImpl, error) { + deploymentConfigurationService := &DeploymentConfigurationServiceImpl{ + logger: logger, + configMapRepository: configMapRepository, + } + + return deploymentConfigurationService, nil +} + +func (impl DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, envId int) (*ConfigDataResponse, error) { + var configDataResponse ConfigDataResponse + var cMCSNamesEnvLevel []chartConfig.CMCSNames + cMCSNamesAppLevel, err := impl.configMapRepository.GetConfigNamesAppLevel(appId) + if err != nil { + return nil, err + } + if envId > 0 { + cMCSNamesEnvLevel, err = impl.configMapRepository.GetConfigNamesEnvLevel(appId, envId) + } + for i, name := range cMCSNamesAppLevel { + configDataResponse.ResourceConfig[i].Name = name.CMName + configDataResponse.ResourceConfig[i].Type = CM + } + for i, name := range cMCSNamesEnvLevel { + configDataResponse.ResourceConfig[i].Name = name.CMName + configDataResponse.ResourceConfig[i].Type = CM + } + + return nil, nil +} diff --git a/pkg/config/bean.go b/pkg/config/bean.go new file mode 100644 index 00000000000..9347a0f5d5e --- /dev/null +++ b/pkg/config/bean.go @@ -0,0 +1,24 @@ +package config + +type ConfigState int + +const ( + PublishedConfigState ConfigState = iota + 2 +) + +type ResourceType string + +const ( + CM ResourceType = "ConfigMap" + CS ResourceType = "Secret" + DeploymentTemplate ResourceType = "Deployment Template" +) + +type ConfigDefinition struct { + Name string `json:"name"` + ConfigState ConfigState `json:"draftState"` + Type ResourceType `json:"type"` +} +type ConfigDataResponse struct { + ResourceConfig []ConfigDefinition `json:"resourceConfig"` +} diff --git a/wire_gen.go b/wire_gen.go index 33907b59b9f..56c71d76403 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -104,6 +104,7 @@ import ( repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess" "github.com/devtron-labs/devtron/pkg/commonService" + "github.com/devtron-labs/devtron/pkg/config" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" @@ -167,11 +168,11 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - config, err := sql.GetConfig() + sqlConfig, err := sql.GetConfig() if err != nil { return nil, err } - db, err := sql.NewDbConnection(config, sugaredLogger) + db, err := sql.NewDbConnection(sqlConfig, sugaredLogger) if err != nil { return nil, err } @@ -816,7 +817,13 @@ func InitializeApp() (*App, error) { return nil, err } proxyRouterImpl := proxy.NewProxyRouterImpl(sugaredLogger, proxyConfig, enforcerImpl) - muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl) + deploymentConfigurationServiceImpl, err := config.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapRepositoryImpl) + if err != nil { + return nil, err + } + deploymentConfigurationRestHandlerImpl := restHandler.NewDeploymentConfigurationRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerUtilImpl, enforcerImpl, deploymentConfigurationServiceImpl) + deploymentConfigurationRouterImpl := router.NewDeploymentConfigurationRouter(deploymentConfigurationRestHandlerImpl) + muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl, deploymentConfigurationRouterImpl) loggingMiddlewareImpl := util4.NewLoggingMiddlewareImpl(userServiceImpl) mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, syncedEnforcer, db, pubSubClientServiceImpl, sessionManager, posthogClient, loggingMiddlewareImpl) return mainApp, nil From 4e3582bf9a3d3b0354228a21d5f79cc7076a06cd Mon Sep 17 00:00:00 2001 From: adi6859 Date: Thu, 1 Feb 2024 23:07:12 +0530 Subject: [PATCH 05/45] story(configDiffView) : WIP "some code changed" --- .../DeploymentConfigurationRestHandler.go | 19 +++++++-------- api/router/DeploymentConfigRouter.go | 5 +++- api/router/router.go | 2 +- .../chartConfig/ConfigMapRepository.go | 2 +- pkg/config/DeploymentConfigurationService.go | 23 +++++++++++++------ 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 09fb33feeaf..7f38b09052b 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -7,7 +7,6 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/config" "github.com/devtron-labs/devtron/util/rbac" - "github.com/gorilla/mux" "go.uber.org/zap" "gopkg.in/go-playground/validator.v9" "net/http" @@ -47,18 +46,16 @@ func (handler *DeploymentConfigurationRestHandlerImpl) ConfigAutoComplete(w http common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - vars := mux.Vars(r) - //todo aditya get from query params - appId, err := strconv.Atoi(vars["appId"]) - if err != nil { - handler.logger.Errorw("request err, CSEnvironmentFetch", "err", err, "appId", appId) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + appIdQueryParam := r.URL.Query().Get("appId") + appId, err := strconv.Atoi(appIdQueryParam) + if appIdQueryParam == "" || err != nil { + common.WriteJsonResp(w, err, "invalid appId", http.StatusBadRequest) return } - envId, err := strconv.Atoi(vars["envId"]) - if err != nil { - handler.logger.Errorw("bad request", "err", err) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + envIdQueryParam := r.URL.Query().Get("appId") + envId, err := strconv.Atoi(appIdQueryParam) + if envIdQueryParam == "" || err != nil { + common.WriteJsonResp(w, err, "invalid appId", http.StatusBadRequest) return } diff --git a/api/router/DeploymentConfigRouter.go b/api/router/DeploymentConfigRouter.go index b79409f9c51..eb07fa804a0 100644 --- a/api/router/DeploymentConfigRouter.go +++ b/api/router/DeploymentConfigRouter.go @@ -22,5 +22,8 @@ func NewDeploymentConfigurationRouter(deploymentGroupRestHandler restHandler.Dep func (router DeploymentConfigurationRouterImpl) initDeploymentConfigurationRouter(configRouter *mux.Router) { configRouter.Path("/autocomplete"). - HandlerFunc(router.deploymentGroupRestHandler.ConfigAutoComplete).Methods("POST") + HandlerFunc(router.deploymentGroupRestHandler.ConfigAutoComplete). + Queries("appId", "{appId}"). + Queries("envId", "{envId}"). + Methods("POST") } diff --git a/api/router/router.go b/api/router/router.go index 8437aa4a720..829a1e158bf 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -313,7 +313,7 @@ func (r MuxRouter) Init() { chartRefRouter := r.Router.PathPrefix("/orchestrator/chartref").Subrouter() r.ChartRefRouter.initChartRefRouter(chartRefRouter) - configRouter := r.Router.PathPrefix("/orchestrator/config/").Subrouter() + configRouter := r.Router.PathPrefix("/orchestrator/config").Subrouter() r.ConfigMapRouter.initConfigMapRouter(configRouter) r.deploymentConfigurationRouter.initDeploymentConfigurationRouter(configRouter) diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index 008f3811a68..3b16f198bf7 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -78,7 +78,7 @@ func (impl ConfigMapRepositoryImpl) GetConfigNamesEnvLevel(appId int, envId int) var cMCSNames []CMCSNames query := "SELECT json_array_elements(config_map_data::json->'maps')->>'name' AS cm_name, " + "json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name " + - "FROM config_map_app_level WHERE app_id = ? AND env_id=?;" + "FROM config_map_env_level WHERE app_id = ? AND env_id=?;" _, err := impl.dbConnection.Query(cMCSNames, query, appId, envId) return cMCSNames, err diff --git a/pkg/config/DeploymentConfigurationService.go b/pkg/config/DeploymentConfigurationService.go index 89f7d0dc7c4..715dccd575a 100644 --- a/pkg/config/DeploymentConfigurationService.go +++ b/pkg/config/DeploymentConfigurationService.go @@ -24,7 +24,7 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, configMapR } func (impl DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, envId int) (*ConfigDataResponse, error) { - var configDataResponse ConfigDataResponse + var configDataResponse *ConfigDataResponse var cMCSNamesEnvLevel []chartConfig.CMCSNames cMCSNamesAppLevel, err := impl.configMapRepository.GetConfigNamesAppLevel(appId) if err != nil { @@ -33,14 +33,23 @@ func (impl DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, env if envId > 0 { cMCSNamesEnvLevel, err = impl.configMapRepository.GetConfigNamesEnvLevel(appId, envId) } - for i, name := range cMCSNamesAppLevel { + configDataResponse = setConfigDataResponse(cMCSNamesAppLevel) + configDataResponse = setConfigDataResponse(cMCSNamesEnvLevel) + + return configDataResponse, nil +} + +func setConfigDataResponse(cMCSNames []chartConfig.CMCSNames) *ConfigDataResponse { + var configDataResponse *ConfigDataResponse + for i, name := range cMCSNames { configDataResponse.ResourceConfig[i].Name = name.CMName configDataResponse.ResourceConfig[i].Type = CM + configDataResponse.ResourceConfig[i].ConfigState = PublishedConfigState } - for i, name := range cMCSNamesEnvLevel { - configDataResponse.ResourceConfig[i].Name = name.CMName - configDataResponse.ResourceConfig[i].Type = CM + for i, name := range cMCSNames { + configDataResponse.ResourceConfig[i].Name = name.CSName + configDataResponse.ResourceConfig[i].Type = CS + configDataResponse.ResourceConfig[i].ConfigState = PublishedConfigState } - - return nil, nil + return configDataResponse } From 8c9a4ccb4e2f7543a9e3e5067f1acb47ce4e0112 Mon Sep 17 00:00:00 2001 From: adi6859 Date: Fri, 2 Feb 2024 16:30:14 +0530 Subject: [PATCH 06/45] story(configDiffView) : support for names added --- .../DeploymentConfigurationRestHandler.go | 17 +++--- api/restHandler/common/ParamParserUtils.go | 30 ++++++++++ api/router/DeploymentConfigRouter.go | 2 +- .../chartConfig/ConfigMapRepository.go | 4 +- pkg/config/DeploymentConfigurationService.go | 58 +++++++++++-------- pkg/config/bean.go | 6 +- pkg/pipeline/ConfigMapService.go | 19 ++++++ specs/configDiffView.yaml | 35 +++++++---- wire_gen.go | 2 +- 9 files changed, 122 insertions(+), 51 deletions(-) create mode 100644 api/restHandler/common/ParamParserUtils.go diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 7f38b09052b..563c32fe0fa 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -10,7 +10,6 @@ import ( "go.uber.org/zap" "gopkg.in/go-playground/validator.v9" "net/http" - "strconv" ) type DeploymentConfigurationRestHandler interface { @@ -46,21 +45,19 @@ func (handler *DeploymentConfigurationRestHandlerImpl) ConfigAutoComplete(w http common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - appIdQueryParam := r.URL.Query().Get("appId") - appId, err := strconv.Atoi(appIdQueryParam) - if appIdQueryParam == "" || err != nil { + appId, err := common.ExtractIntQueryParam(w, r, "appId", nil) + if err != nil { common.WriteJsonResp(w, err, "invalid appId", http.StatusBadRequest) return } - envIdQueryParam := r.URL.Query().Get("appId") - envId, err := strconv.Atoi(appIdQueryParam) - if envIdQueryParam == "" || err != nil { - common.WriteJsonResp(w, err, "invalid appId", http.StatusBadRequest) + envId, err := common.ExtractIntQueryParam(w, r, "envId", nil) + if err != nil { + common.WriteJsonResp(w, err, "invalid envId", http.StatusBadRequest) return } //RBAC START - token := r.Header.Get("token") + token := r.Header.Get(common.TokenHeaderKey) object := handler.enforcerUtil.GetAppRBACNameByAppId(appId) ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, casbin.ActionGet) if !ok { @@ -71,7 +68,7 @@ func (handler *DeploymentConfigurationRestHandlerImpl) ConfigAutoComplete(w http res, err := handler.deploymentConfigurationService.ConfigAutoComplete(appId, envId) if err != nil { - handler.logger.Errorw("service err, CSEnvironmentFetch", "err", err, "appId", appId) + handler.logger.Errorw("service err, ConfigAutoComplete ", "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } diff --git a/api/restHandler/common/ParamParserUtils.go b/api/restHandler/common/ParamParserUtils.go new file mode 100644 index 00000000000..687ff89c677 --- /dev/null +++ b/api/restHandler/common/ParamParserUtils.go @@ -0,0 +1,30 @@ +package common + +import ( + "net/http" + "strconv" +) + +const TokenHeaderKey = "token" + +func ExtractIntQueryParam(w http.ResponseWriter, r *http.Request, paramName string, defaultVal *int) (int, error) { + queryParams := r.URL.Query() + paramValue := queryParams.Get(paramName) + if len(paramValue) == 0 { + return *defaultVal, nil + } + paramIntValue, err := convertToInt(w, paramValue) + if err != nil { + return 0, err + } + return paramIntValue, nil +} + +func convertToInt(w http.ResponseWriter, paramValue string) (int, error) { + paramIntValue, err := strconv.Atoi(paramValue) + if err != nil { + WriteJsonResp(w, err, nil, http.StatusBadRequest) + return 0, err + } + return paramIntValue, nil +} diff --git a/api/router/DeploymentConfigRouter.go b/api/router/DeploymentConfigRouter.go index eb07fa804a0..6792e0c1fe3 100644 --- a/api/router/DeploymentConfigRouter.go +++ b/api/router/DeploymentConfigRouter.go @@ -25,5 +25,5 @@ func (router DeploymentConfigurationRouterImpl) initDeploymentConfigurationRoute HandlerFunc(router.deploymentGroupRestHandler.ConfigAutoComplete). Queries("appId", "{appId}"). Queries("envId", "{envId}"). - Methods("POST") + Methods("GET") } diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index 3b16f198bf7..7dad328fedb 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -71,7 +71,7 @@ func (impl ConfigMapRepositoryImpl) GetConfigNamesAppLevel(appId int) ([]CMCSNam "json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name " + "FROM config_map_app_level WHERE app_id = ?;" - _, err := impl.dbConnection.Query(cMCSNames, query, appId) + _, err := impl.dbConnection.Query(&cMCSNames, query, appId) return cMCSNames, err } func (impl ConfigMapRepositoryImpl) GetConfigNamesEnvLevel(appId int, envId int) ([]CMCSNames, error) { @@ -80,7 +80,7 @@ func (impl ConfigMapRepositoryImpl) GetConfigNamesEnvLevel(appId int, envId int) "json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name " + "FROM config_map_env_level WHERE app_id = ? AND env_id=?;" - _, err := impl.dbConnection.Query(cMCSNames, query, appId, envId) + _, err := impl.dbConnection.Query(&cMCSNames, query, appId, envId) return cMCSNames, err } diff --git a/pkg/config/DeploymentConfigurationService.go b/pkg/config/DeploymentConfigurationService.go index 715dccd575a..f1e28624065 100644 --- a/pkg/config/DeploymentConfigurationService.go +++ b/pkg/config/DeploymentConfigurationService.go @@ -2,6 +2,7 @@ package config import ( "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "github.com/devtron-labs/devtron/pkg/pipeline" "go.uber.org/zap" ) @@ -10,46 +11,57 @@ type DeploymentConfigurationService interface { } type DeploymentConfigurationServiceImpl struct { - logger *zap.SugaredLogger - configMapRepository chartConfig.ConfigMapRepository + logger *zap.SugaredLogger + configMapService pipeline.ConfigMapService } -func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, configMapRepository chartConfig.ConfigMapRepository) (*DeploymentConfigurationServiceImpl, error) { +func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, + configMapService pipeline.ConfigMapService, +) (*DeploymentConfigurationServiceImpl, error) { deploymentConfigurationService := &DeploymentConfigurationServiceImpl{ - logger: logger, - configMapRepository: configMapRepository, + logger: logger, + configMapService: configMapService, } return deploymentConfigurationService, nil } -func (impl DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, envId int) (*ConfigDataResponse, error) { +func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, envId int) (*ConfigDataResponse, error) { var configDataResponse *ConfigDataResponse - var cMCSNamesEnvLevel []chartConfig.CMCSNames - cMCSNamesAppLevel, err := impl.configMapRepository.GetConfigNamesAppLevel(appId) + cMCSNamesAppLevel, cMCSNamesEnvLevel, err := impl.configMapService.FetchCmCsNamesAppAndEnvLevel(appId, envId) if err != nil { return nil, err } - if envId > 0 { - cMCSNamesEnvLevel, err = impl.configMapRepository.GetConfigNamesEnvLevel(appId, envId) - } - configDataResponse = setConfigDataResponse(cMCSNamesAppLevel) - configDataResponse = setConfigDataResponse(cMCSNamesEnvLevel) + configDataResponse = setConfigDataResponse(cMCSNamesAppLevel, configDataResponse) + configDataResponse = setConfigDataResponse(cMCSNamesEnvLevel, configDataResponse) return configDataResponse, nil } -func setConfigDataResponse(cMCSNames []chartConfig.CMCSNames) *ConfigDataResponse { - var configDataResponse *ConfigDataResponse - for i, name := range cMCSNames { - configDataResponse.ResourceConfig[i].Name = name.CMName - configDataResponse.ResourceConfig[i].Type = CM - configDataResponse.ResourceConfig[i].ConfigState = PublishedConfigState +func setConfigDataResponse(cMCSNames []chartConfig.CMCSNames, configDataResponse *ConfigDataResponse) *ConfigDataResponse { + if cMCSNames == nil { + return configDataResponse } - for i, name := range cMCSNames { - configDataResponse.ResourceConfig[i].Name = name.CSName - configDataResponse.ResourceConfig[i].Type = CS - configDataResponse.ResourceConfig[i].ConfigState = PublishedConfigState + configDataResponse = &ConfigDataResponse{} + for _, name := range cMCSNames { + if name.CMName != "" { + // Fill in CM data if the CMName is not empty + cmConfig := setConfigProperty(name.CMName, CM, PublishedConfigState) + configDataResponse.ResourceConfig = append(configDataResponse.ResourceConfig, cmConfig) + } + if name.CSName != "" { + // Fill in CS data if the CSName is not empty + csConfig := setConfigProperty(name.CSName, CS, PublishedConfigState) + configDataResponse.ResourceConfig = append(configDataResponse.ResourceConfig, csConfig) + } + } return configDataResponse } +func setConfigProperty(name string, configType ResourceType, State ConfigState) ConfigProperty { + return ConfigProperty{ + Name: name, + Type: configType, + ConfigState: State, + } +} diff --git a/pkg/config/bean.go b/pkg/config/bean.go index 9347a0f5d5e..69fc6aae7ca 100644 --- a/pkg/config/bean.go +++ b/pkg/config/bean.go @@ -14,11 +14,11 @@ const ( DeploymentTemplate ResourceType = "Deployment Template" ) -type ConfigDefinition struct { +type ConfigProperty struct { Name string `json:"name"` - ConfigState ConfigState `json:"draftState"` + ConfigState ConfigState `json:"configState"` Type ResourceType `json:"type"` } type ConfigDataResponse struct { - ResourceConfig []ConfigDefinition `json:"resourceConfig"` + ResourceConfig []ConfigProperty `json:"resourceConfig"` } diff --git a/pkg/pipeline/ConfigMapService.go b/pkg/pipeline/ConfigMapService.go index d291531a0ac..22e32179603 100644 --- a/pkg/pipeline/ConfigMapService.go +++ b/pkg/pipeline/ConfigMapService.go @@ -82,6 +82,8 @@ type ConfigMapService interface { ConfigSecretEnvironmentDelete(createJobEnvOverrideRequest *bean.CreateJobEnvOverridePayload) (*bean.CreateJobEnvOverridePayload, error) ConfigSecretEnvironmentGet(appId int) ([]bean.JobEnvOverrideResponse, error) ConfigSecretEnvironmentClone(appId int, cloneAppId int, userId int32) ([]chartConfig.ConfigMapEnvModel, error) + + FetchCmCsNamesAppAndEnvLevel(appId int, envId int) ([]chartConfig.CMCSNames, []chartConfig.CMCSNames, error) } type ConfigMapServiceImpl struct { @@ -1992,3 +1994,20 @@ func (impl ConfigMapServiceImpl) ConfigSecretEnvironmentClone(appId int, cloneAp return jobEnvOverrideResponse, nil } +func (impl ConfigMapServiceImpl) FetchCmCsNamesAppAndEnvLevel(appId int, envId int) ([]chartConfig.CMCSNames, []chartConfig.CMCSNames, error) { + var cMCSNamesEnvLevel []chartConfig.CMCSNames + cMCSNamesAppLevel, err := impl.configMapRepository.GetConfigNamesAppLevel(appId) + if err != nil { + impl.logger.Errorw("error in getting CM/CS nmaes at app level ", "appId", appId, "err", err) + return nil, nil, err + } + if envId > 0 { + cMCSNamesEnvLevel, err = impl.configMapRepository.GetConfigNamesEnvLevel(appId, envId) + if err != nil { + impl.logger.Errorw("error in getting CM/CS names at env level ", "appId", appId, "err", err) + + return nil, nil, err + } + } + return cMCSNamesAppLevel, cMCSNamesEnvLevel, nil +} diff --git a/specs/configDiffView.yaml b/specs/configDiffView.yaml index 4f44994fcae..065fd1ec3f5 100644 --- a/specs/configDiffView.yaml +++ b/specs/configDiffView.yaml @@ -27,7 +27,7 @@ paths: schema: type: array items: - $ref: "#/components/schemas/ResourceConfig" + $ref: "#/components/schemas/ConfigProperty" '500': description: will get this response if any failure occurs at server side. @@ -40,21 +40,34 @@ paths: components: schemas: - ResourceConfig: + ConfigDataResponse: + type: object + properties: + resourceConfig: + type: array + items: + $ref: '#/components/schemas/ConfigProperty' + + ConfigProperty: type: object properties: name: type: string - description: Name of config + description: Name of the config example: cm-1 nullable: true configState: - type: integer - description: state of config{1 represent published state ,2 represents draft state , 3 represent approval pending state } - example: 1 - nullable: false + $ref: '#/components/schemas/ConfigStateEnum' type: - type: string - description: describe the config type {possible values are ConfigMap, Secret, Deployment Template} - example: ConfigMap - nullable: true \ No newline at end of file + $ref: '#/components/schemas/ResourceTypeEnum' + + ConfigStateEnum: + type: integer + enum: [ 1, 2, 3 ] + description: State of config (1 represents published state, 2 represents draft state, 3 represents approval pending state) + + ResourceTypeEnum: + type: string + enum: [ "ConfigMap", "Secret", "Deployment Template" ] + description: Describe the config type (possible values are ConfigMap, Secret, Deployment Template) + diff --git a/wire_gen.go b/wire_gen.go index 56c71d76403..f587e4d80f7 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -817,7 +817,7 @@ func InitializeApp() (*App, error) { return nil, err } proxyRouterImpl := proxy.NewProxyRouterImpl(sugaredLogger, proxyConfig, enforcerImpl) - deploymentConfigurationServiceImpl, err := config.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapRepositoryImpl) + deploymentConfigurationServiceImpl, err := config.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl) if err != nil { return nil, err } From 55a2c0c5aa3e8009eb6964f9a0eb63866be59d6a Mon Sep 17 00:00:00 2001 From: adi6859 Date: Fri, 2 Feb 2024 16:34:49 +0530 Subject: [PATCH 07/45] story(configDiffView) : iota removed --- pkg/config/bean.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/bean.go b/pkg/config/bean.go index 69fc6aae7ca..1539f71d9e3 100644 --- a/pkg/config/bean.go +++ b/pkg/config/bean.go @@ -3,7 +3,7 @@ package config type ConfigState int const ( - PublishedConfigState ConfigState = iota + 2 + PublishedConfigState ConfigState = 3 ) type ResourceType string From bcdfa26023d0c02ba2c5df516aeb91ad2bcee8ce Mon Sep 17 00:00:00 2001 From: adi6859 Date: Fri, 2 Feb 2024 17:39:09 +0530 Subject: [PATCH 08/45] story(configDiffView) : pg no rows handled --- .../sql/repository/chartConfig/ConfigMapRepository.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index 7dad328fedb..c1ab4f0ba28 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -72,6 +72,11 @@ func (impl ConfigMapRepositoryImpl) GetConfigNamesAppLevel(appId int) ([]CMCSNam "FROM config_map_app_level WHERE app_id = ?;" _, err := impl.dbConnection.Query(&cMCSNames, query, appId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error occurred while fetching CM/CS names ", "appId", appId, "err", err) + } else { + err = nil //ignoring noRows Error + } return cMCSNames, err } func (impl ConfigMapRepositoryImpl) GetConfigNamesEnvLevel(appId int, envId int) ([]CMCSNames, error) { @@ -81,6 +86,11 @@ func (impl ConfigMapRepositoryImpl) GetConfigNamesEnvLevel(appId int, envId int) "FROM config_map_env_level WHERE app_id = ? AND env_id=?;" _, err := impl.dbConnection.Query(&cMCSNames, query, appId, envId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error occurred while fetching CM/CS names ", "appId", appId, "envId", envId, "err", err) + } else { + err = nil //ignoring noRows Error + } return cMCSNames, err } From 197db9bdba4e2866ecb9a8bf701e62c569514a5c Mon Sep 17 00:00:00 2001 From: adi6859 Date: Fri, 2 Feb 2024 17:41:55 +0530 Subject: [PATCH 09/45] story(configDiffView) : spelling check --- pkg/pipeline/ConfigMapService.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/pipeline/ConfigMapService.go b/pkg/pipeline/ConfigMapService.go index 22e32179603..2f2ae34fb48 100644 --- a/pkg/pipeline/ConfigMapService.go +++ b/pkg/pipeline/ConfigMapService.go @@ -1996,9 +1996,10 @@ func (impl ConfigMapServiceImpl) ConfigSecretEnvironmentClone(appId int, cloneAp } func (impl ConfigMapServiceImpl) FetchCmCsNamesAppAndEnvLevel(appId int, envId int) ([]chartConfig.CMCSNames, []chartConfig.CMCSNames, error) { var cMCSNamesEnvLevel []chartConfig.CMCSNames + cMCSNamesAppLevel, err := impl.configMapRepository.GetConfigNamesAppLevel(appId) if err != nil { - impl.logger.Errorw("error in getting CM/CS nmaes at app level ", "appId", appId, "err", err) + impl.logger.Errorw("error in getting CM/CS names at app level ", "appId", appId, "err", err) return nil, nil, err } if envId > 0 { From 7469461216ac1356434284d529d7ff1728b1ac5f Mon Sep 17 00:00:00 2001 From: adi6859 Date: Mon, 5 Feb 2024 14:29:30 +0530 Subject: [PATCH 10/45] story(configDiffView) : code review comment resolved --- .../DeploymentConfigurationRestHandler.go | 3 -- .../chartConfig/ConfigMapRepository.go | 47 +++++++++++-------- pkg/config/DeploymentConfigurationService.go | 38 ++++++++------- wire_gen.go | 2 +- 4 files changed, 48 insertions(+), 42 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 563c32fe0fa..3dc52c7f6ea 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -20,21 +20,18 @@ type DeploymentConfigurationRestHandlerImpl struct { userAuthService user.UserService validator *validator.Validate enforcerUtil rbac.EnforcerUtil - enforcer casbin.Enforcer deploymentConfigurationService config.DeploymentConfigurationService } func NewDeploymentConfigurationRestHandlerImpl(logger *zap.SugaredLogger, userAuthService user.UserService, enforcerUtil rbac.EnforcerUtil, - enforcer casbin.Enforcer, deploymentConfigurationService config.DeploymentConfigurationService, ) *DeploymentConfigurationRestHandlerImpl { return &DeploymentConfigurationRestHandlerImpl{ logger: logger, userAuthService: userAuthService, enforcerUtil: enforcerUtil, - enforcer: enforcer, deploymentConfigurationService: deploymentConfigurationService, } } diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index c1ab4f0ba28..0b614f252b4 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -61,37 +61,44 @@ type ConfigMapAppModel struct { sql.AuditLog } type CMCSNames struct { + //TableName struct{} `sql:"config_map_app_level" pg:",discard_unknown_columns"` CMName string `json:"cm_name"` CSName string `json:"cs_name"` } func (impl ConfigMapRepositoryImpl) GetConfigNamesAppLevel(appId int) ([]CMCSNames, error) { var cMCSNames []CMCSNames - query := "SELECT json_array_elements(config_map_data::json->'maps')->>'name' AS cm_name, " + - "json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name " + - "FROM config_map_app_level WHERE app_id = ?;" - - _, err := impl.dbConnection.Query(&cMCSNames, query, appId) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("error occurred while fetching CM/CS names ", "appId", appId, "err", err) - } else { - err = nil //ignoring noRows Error + query := impl.dbConnection. + Model(). + Table("config_map_app_level"). + ColumnExpr("json_array_elements(config_map_data::json->'maps')->>'name' AS cm_name"). + ColumnExpr("json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name"). + Where("app_id = ?", appId) + + if err := query.Select(&cMCSNames); err != nil { + if err != pg.ErrNoRows { + impl.Logger.Errorw("error occurred while fetching CM/CS names", "appId", appId, "err", err) + } + return cMCSNames, err } - return cMCSNames, err + return cMCSNames, nil } func (impl ConfigMapRepositoryImpl) GetConfigNamesEnvLevel(appId int, envId int) ([]CMCSNames, error) { var cMCSNames []CMCSNames - query := "SELECT json_array_elements(config_map_data::json->'maps')->>'name' AS cm_name, " + - "json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name " + - "FROM config_map_env_level WHERE app_id = ? AND env_id=?;" - - _, err := impl.dbConnection.Query(&cMCSNames, query, appId, envId) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("error occurred while fetching CM/CS names ", "appId", appId, "envId", envId, "err", err) - } else { - err = nil //ignoring noRows Error + query := impl.dbConnection. + Model(). + Table("config_map_env_level"). + ColumnExpr("json_array_elements(config_map_data::json->'maps')->>'name' AS cm_name"). + ColumnExpr("json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name"). + Where("app_id = ?", appId) + + if err := query.Select(&cMCSNames); err != nil { + if err != pg.ErrNoRows { + impl.Logger.Errorw("error occurred while fetching CM/CS names", "appId", appId, "err", err) + } + return cMCSNames, err } - return cMCSNames, err + return cMCSNames, nil } func (impl ConfigMapRepositoryImpl) CreateAppLevel(model *ConfigMapAppModel) (*ConfigMapAppModel, error) { diff --git a/pkg/config/DeploymentConfigurationService.go b/pkg/config/DeploymentConfigurationService.go index f1e28624065..0fada596c29 100644 --- a/pkg/config/DeploymentConfigurationService.go +++ b/pkg/config/DeploymentConfigurationService.go @@ -27,38 +27,40 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, } func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, envId int) (*ConfigDataResponse, error) { - var configDataResponse *ConfigDataResponse cMCSNamesAppLevel, cMCSNamesEnvLevel, err := impl.configMapService.FetchCmCsNamesAppAndEnvLevel(appId, envId) if err != nil { return nil, err } - configDataResponse = setConfigDataResponse(cMCSNamesAppLevel, configDataResponse) - configDataResponse = setConfigDataResponse(cMCSNamesEnvLevel, configDataResponse) + combinedProperties := make([]ConfigProperty, 0) + //App level cm/cs + combinedProperties = append(combinedProperties, getConfigPropertyList(cMCSNamesAppLevel)...) + //env level cm/cs + combinedProperties = append(combinedProperties, getConfigPropertyList(cMCSNamesEnvLevel)...) + //DT + combinedProperties = append(combinedProperties, getConfigProperty("", DeploymentTemplate, PublishedConfigState)) + combinedProperties = append(combinedProperties) - return configDataResponse, nil + return &ConfigDataResponse{ResourceConfig: combinedProperties}, nil } - -func setConfigDataResponse(cMCSNames []chartConfig.CMCSNames, configDataResponse *ConfigDataResponse) *ConfigDataResponse { - if cMCSNames == nil { - return configDataResponse +func getConfigPropertyList(cMCSNames []chartConfig.CMCSNames) []ConfigProperty { + properties := make([]ConfigProperty, 0) + if len(cMCSNames) == 0 { + return properties } - configDataResponse = &ConfigDataResponse{} for _, name := range cMCSNames { + // Fill in CM property if the CMName is not empty if name.CMName != "" { - // Fill in CM data if the CMName is not empty - cmConfig := setConfigProperty(name.CMName, CM, PublishedConfigState) - configDataResponse.ResourceConfig = append(configDataResponse.ResourceConfig, cmConfig) + properties = append(properties, getConfigProperty(name.CMName, CM, PublishedConfigState)) } + // Fill in CS property if the CSName is not empty if name.CSName != "" { - // Fill in CS data if the CSName is not empty - csConfig := setConfigProperty(name.CSName, CS, PublishedConfigState) - configDataResponse.ResourceConfig = append(configDataResponse.ResourceConfig, csConfig) - } + properties = append(properties, getConfigProperty(name.CSName, CS, PublishedConfigState)) + } } - return configDataResponse + return properties } -func setConfigProperty(name string, configType ResourceType, State ConfigState) ConfigProperty { +func getConfigProperty(name string, configType ResourceType, State ConfigState) ConfigProperty { return ConfigProperty{ Name: name, Type: configType, diff --git a/wire_gen.go b/wire_gen.go index f587e4d80f7..7dd53c09519 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -821,7 +821,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - deploymentConfigurationRestHandlerImpl := restHandler.NewDeploymentConfigurationRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerUtilImpl, enforcerImpl, deploymentConfigurationServiceImpl) + deploymentConfigurationRestHandlerImpl := restHandler.NewDeploymentConfigurationRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerUtilImpl, deploymentConfigurationServiceImpl) deploymentConfigurationRouterImpl := router.NewDeploymentConfigurationRouter(deploymentConfigurationRestHandlerImpl) muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl, deploymentConfigurationRouterImpl) loggingMiddlewareImpl := util4.NewLoggingMiddlewareImpl(userServiceImpl) From c7a145a1db8582c600f79115d26a705ab10958b0 Mon Sep 17 00:00:00 2001 From: adi6859 Date: Mon, 5 Feb 2024 14:31:23 +0530 Subject: [PATCH 11/45] story(configDiffView) : env id added --- internal/sql/repository/chartConfig/ConfigMapRepository.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index 0b614f252b4..fbc9c8554a0 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -90,7 +90,8 @@ func (impl ConfigMapRepositoryImpl) GetConfigNamesEnvLevel(appId int, envId int) Table("config_map_env_level"). ColumnExpr("json_array_elements(config_map_data::json->'maps')->>'name' AS cm_name"). ColumnExpr("json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name"). - Where("app_id = ?", appId) + Where("app_id = ?", appId). + Where("environment_id=?", envId) if err := query.Select(&cMCSNames); err != nil { if err != pg.ErrNoRows { From 4eaa13888a46c02cd0b8d7914427749e9bdaec7b Mon Sep 17 00:00:00 2001 From: adi6859 Date: Mon, 5 Feb 2024 16:14:37 +0530 Subject: [PATCH 12/45] story(configDiffView) : intersection added --- pkg/config/DeploymentConfigurationService.go | 22 ++++++++++++-------- pkg/config/bean.go | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/config/DeploymentConfigurationService.go b/pkg/config/DeploymentConfigurationService.go index 0fada596c29..cbf62eeb323 100644 --- a/pkg/config/DeploymentConfigurationService.go +++ b/pkg/config/DeploymentConfigurationService.go @@ -31,37 +31,41 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en if err != nil { return nil, err } - combinedProperties := make([]ConfigProperty, 0) + combinedProperties := make([]*ConfigProperty, 0) //App level cm/cs - combinedProperties = append(combinedProperties, getConfigPropertyList(cMCSNamesAppLevel)...) + combinedProperties = append(combinedProperties, getConfigPropertyList(cMCSNamesAppLevel, combinedProperties)...) //env level cm/cs - combinedProperties = append(combinedProperties, getConfigPropertyList(cMCSNamesEnvLevel)...) + combinedProperties = append(combinedProperties, getConfigPropertyList(cMCSNamesEnvLevel, combinedProperties)...) //DT combinedProperties = append(combinedProperties, getConfigProperty("", DeploymentTemplate, PublishedConfigState)) combinedProperties = append(combinedProperties) return &ConfigDataResponse{ResourceConfig: combinedProperties}, nil } -func getConfigPropertyList(cMCSNames []chartConfig.CMCSNames) []ConfigProperty { - properties := make([]ConfigProperty, 0) +func getConfigPropertyList(cMCSNames []chartConfig.CMCSNames, combinedProperties []*ConfigProperty) []*ConfigProperty { + properties := make([]*ConfigProperty, 0) if len(cMCSNames) == 0 { return properties } + combinedNames := make(map[string]bool) + for _, config := range combinedProperties { + combinedNames[config.Name] = true + } for _, name := range cMCSNames { // Fill in CM property if the CMName is not empty - if name.CMName != "" { + if name.CMName != "" && !combinedNames[name.CMName] { properties = append(properties, getConfigProperty(name.CMName, CM, PublishedConfigState)) } // Fill in CS property if the CSName is not empty - if name.CSName != "" { + if name.CSName != "" && !combinedNames[name.CSName] { properties = append(properties, getConfigProperty(name.CSName, CS, PublishedConfigState)) } } return properties } -func getConfigProperty(name string, configType ResourceType, State ConfigState) ConfigProperty { - return ConfigProperty{ +func getConfigProperty(name string, configType ResourceType, State ConfigState) *ConfigProperty { + return &ConfigProperty{ Name: name, Type: configType, ConfigState: State, diff --git a/pkg/config/bean.go b/pkg/config/bean.go index 1539f71d9e3..6496e22151d 100644 --- a/pkg/config/bean.go +++ b/pkg/config/bean.go @@ -20,5 +20,5 @@ type ConfigProperty struct { Type ResourceType `json:"type"` } type ConfigDataResponse struct { - ResourceConfig []ConfigProperty `json:"resourceConfig"` + ResourceConfig []*ConfigProperty `json:"resourceConfig"` } From 256e505d48a98570d15b3ca3217e9c1cae5c4fb9 Mon Sep 17 00:00:00 2001 From: adi6859 Date: Mon, 5 Feb 2024 17:40:49 +0530 Subject: [PATCH 13/45] story(configDiffView) : comments removed --- pkg/config/DeploymentConfigurationService.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pkg/config/DeploymentConfigurationService.go b/pkg/config/DeploymentConfigurationService.go index cbf62eeb323..c652ce3b554 100644 --- a/pkg/config/DeploymentConfigurationService.go +++ b/pkg/config/DeploymentConfigurationService.go @@ -32,17 +32,16 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en return nil, err } combinedProperties := make([]*ConfigProperty, 0) - //App level cm/cs - combinedProperties = append(combinedProperties, getConfigPropertyList(cMCSNamesAppLevel, combinedProperties)...) - //env level cm/cs - combinedProperties = append(combinedProperties, getConfigPropertyList(cMCSNamesEnvLevel, combinedProperties)...) - //DT - combinedProperties = append(combinedProperties, getConfigProperty("", DeploymentTemplate, PublishedConfigState)) + combinedProperties = append(combinedProperties, + getUniqueConfigPropertyList(cMCSNamesAppLevel, combinedProperties)...) + combinedProperties = append(combinedProperties, + getUniqueConfigPropertyList(cMCSNamesEnvLevel, combinedProperties)...) + combinedProperties = append(combinedProperties, + getConfigProperty("", DeploymentTemplate, PublishedConfigState)) combinedProperties = append(combinedProperties) - return &ConfigDataResponse{ResourceConfig: combinedProperties}, nil } -func getConfigPropertyList(cMCSNames []chartConfig.CMCSNames, combinedProperties []*ConfigProperty) []*ConfigProperty { +func getUniqueConfigPropertyList(cMCSNames []chartConfig.CMCSNames, combinedProperties []*ConfigProperty) []*ConfigProperty { properties := make([]*ConfigProperty, 0) if len(cMCSNames) == 0 { return properties @@ -59,7 +58,6 @@ func getConfigPropertyList(cMCSNames []chartConfig.CMCSNames, combinedProperties // Fill in CS property if the CSName is not empty if name.CSName != "" && !combinedNames[name.CSName] { properties = append(properties, getConfigProperty(name.CSName, CS, PublishedConfigState)) - } } return properties From 47f798ffa0a8fb8522e99d475ebee529d6073a6d Mon Sep 17 00:00:00 2001 From: adi6859 Date: Tue, 6 Feb 2024 14:41:25 +0530 Subject: [PATCH 14/45] story(configDiffView) : code review comment resolved --- .../chartConfig/ConfigMapRepository.go | 30 ++++++------------- pkg/config/DeploymentConfigurationService.go | 3 +- pkg/pipeline/ConfigMapService.go | 7 ++--- 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index fbc9c8554a0..6445cf4c772 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -39,8 +39,7 @@ type ConfigMapRepository interface { GetByAppIdAppLevel(appId int) (*ConfigMapAppModel, error) GetByAppIdAndEnvIdEnvLevel(appId int, envId int) (*ConfigMapEnvModel, error) GetEnvLevelByAppId(appId int) ([]*ConfigMapEnvModel, error) - GetConfigNamesAppLevel(appId int) ([]CMCSNames, error) - GetConfigNamesEnvLevel(appId int, envId int) ([]CMCSNames, error) + GetConfigNamesForAppAndEnvLevel(appId int, envId int) ([]CMCSNames, error) } type ConfigMapRepositoryImpl struct { @@ -66,33 +65,22 @@ type CMCSNames struct { CSName string `json:"cs_name"` } -func (impl ConfigMapRepositoryImpl) GetConfigNamesAppLevel(appId int) ([]CMCSNames, error) { +func (impl ConfigMapRepositoryImpl) GetConfigNamesForAppAndEnvLevel(appId int, envId int) ([]CMCSNames, error) { var cMCSNames []CMCSNames + tableName := "config_map_env_level" + if envId == -1 { + tableName = "config_map_app_level" + } query := impl.dbConnection. Model(). - Table("config_map_app_level"). + Table(tableName). ColumnExpr("json_array_elements(config_map_data::json->'maps')->>'name' AS cm_name"). ColumnExpr("json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name"). Where("app_id = ?", appId) - if err := query.Select(&cMCSNames); err != nil { - if err != pg.ErrNoRows { - impl.Logger.Errorw("error occurred while fetching CM/CS names", "appId", appId, "err", err) - } - return cMCSNames, err + if envId > 0 { + query = query.Where("environment_id=?", envId) } - return cMCSNames, nil -} -func (impl ConfigMapRepositoryImpl) GetConfigNamesEnvLevel(appId int, envId int) ([]CMCSNames, error) { - var cMCSNames []CMCSNames - query := impl.dbConnection. - Model(). - Table("config_map_env_level"). - ColumnExpr("json_array_elements(config_map_data::json->'maps')->>'name' AS cm_name"). - ColumnExpr("json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name"). - Where("app_id = ?", appId). - Where("environment_id=?", envId) - if err := query.Select(&cMCSNames); err != nil { if err != pg.ErrNoRows { impl.Logger.Errorw("error occurred while fetching CM/CS names", "appId", appId, "err", err) diff --git a/pkg/config/DeploymentConfigurationService.go b/pkg/config/DeploymentConfigurationService.go index c652ce3b554..f2fe192912b 100644 --- a/pkg/config/DeploymentConfigurationService.go +++ b/pkg/config/DeploymentConfigurationService.go @@ -38,9 +38,9 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en getUniqueConfigPropertyList(cMCSNamesEnvLevel, combinedProperties)...) combinedProperties = append(combinedProperties, getConfigProperty("", DeploymentTemplate, PublishedConfigState)) - combinedProperties = append(combinedProperties) return &ConfigDataResponse{ResourceConfig: combinedProperties}, nil } + func getUniqueConfigPropertyList(cMCSNames []chartConfig.CMCSNames, combinedProperties []*ConfigProperty) []*ConfigProperty { properties := make([]*ConfigProperty, 0) if len(cMCSNames) == 0 { @@ -62,6 +62,7 @@ func getUniqueConfigPropertyList(cMCSNames []chartConfig.CMCSNames, combinedProp } return properties } + func getConfigProperty(name string, configType ResourceType, State ConfigState) *ConfigProperty { return &ConfigProperty{ Name: name, diff --git a/pkg/pipeline/ConfigMapService.go b/pkg/pipeline/ConfigMapService.go index 2f2ae34fb48..0e33e9d8773 100644 --- a/pkg/pipeline/ConfigMapService.go +++ b/pkg/pipeline/ConfigMapService.go @@ -1997,16 +1997,15 @@ func (impl ConfigMapServiceImpl) ConfigSecretEnvironmentClone(appId int, cloneAp func (impl ConfigMapServiceImpl) FetchCmCsNamesAppAndEnvLevel(appId int, envId int) ([]chartConfig.CMCSNames, []chartConfig.CMCSNames, error) { var cMCSNamesEnvLevel []chartConfig.CMCSNames - cMCSNamesAppLevel, err := impl.configMapRepository.GetConfigNamesAppLevel(appId) + cMCSNamesAppLevel, err := impl.configMapRepository.GetConfigNamesForAppAndEnvLevel(appId, -1) if err != nil { impl.logger.Errorw("error in getting CM/CS names at app level ", "appId", appId, "err", err) return nil, nil, err } if envId > 0 { - cMCSNamesEnvLevel, err = impl.configMapRepository.GetConfigNamesEnvLevel(appId, envId) + cMCSNamesEnvLevel, err = impl.configMapRepository.GetConfigNamesForAppAndEnvLevel(appId, envId) if err != nil { - impl.logger.Errorw("error in getting CM/CS names at env level ", "appId", appId, "err", err) - + impl.logger.Errorw("error in getting CM/CS names at env level ", "appId", appId, "envId", envId, "err", err) return nil, nil, err } } From de50f02207a5df692b7a92d3e5409825deb3b774 Mon Sep 17 00:00:00 2001 From: adi6859 Date: Tue, 6 Feb 2024 14:42:54 +0530 Subject: [PATCH 15/45] story(configDiffView) : comment removed --- internal/sql/repository/chartConfig/ConfigMapRepository.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index 6445cf4c772..54b3b7a63c9 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -60,7 +60,6 @@ type ConfigMapAppModel struct { sql.AuditLog } type CMCSNames struct { - //TableName struct{} `sql:"config_map_app_level" pg:",discard_unknown_columns"` CMName string `json:"cm_name"` CSName string `json:"cs_name"` } From f2947e3fc66d1ce617ee973f2774be3ba48c1c35 Mon Sep 17 00:00:00 2001 From: adi6859 Date: Tue, 6 Feb 2024 14:47:41 +0530 Subject: [PATCH 16/45] story(configDiffView) : CMCSNames DTO moved --- .../sql/repository/chartConfig/ConfigMapRepository.go | 11 ++++------- pkg/config/DeploymentConfigurationService.go | 4 ++-- pkg/pipeline/ConfigMapService.go | 6 +++--- pkg/pipeline/bean/ConfigMapBean.go | 4 ++++ 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index 54b3b7a63c9..c593e72ce29 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -18,6 +18,7 @@ package chartConfig import ( + "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" "github.com/go-pg/pg/orm" @@ -39,7 +40,7 @@ type ConfigMapRepository interface { GetByAppIdAppLevel(appId int) (*ConfigMapAppModel, error) GetByAppIdAndEnvIdEnvLevel(appId int, envId int) (*ConfigMapEnvModel, error) GetEnvLevelByAppId(appId int) ([]*ConfigMapEnvModel, error) - GetConfigNamesForAppAndEnvLevel(appId int, envId int) ([]CMCSNames, error) + GetConfigNamesForAppAndEnvLevel(appId int, envId int) ([]bean.CMCSNames, error) } type ConfigMapRepositoryImpl struct { @@ -59,13 +60,9 @@ type ConfigMapAppModel struct { SecretData string `sql:"secret_data"` sql.AuditLog } -type CMCSNames struct { - CMName string `json:"cm_name"` - CSName string `json:"cs_name"` -} -func (impl ConfigMapRepositoryImpl) GetConfigNamesForAppAndEnvLevel(appId int, envId int) ([]CMCSNames, error) { - var cMCSNames []CMCSNames +func (impl ConfigMapRepositoryImpl) GetConfigNamesForAppAndEnvLevel(appId int, envId int) ([]bean.CMCSNames, error) { + var cMCSNames []bean.CMCSNames tableName := "config_map_env_level" if envId == -1 { tableName = "config_map_app_level" diff --git a/pkg/config/DeploymentConfigurationService.go b/pkg/config/DeploymentConfigurationService.go index f2fe192912b..9002ff45646 100644 --- a/pkg/config/DeploymentConfigurationService.go +++ b/pkg/config/DeploymentConfigurationService.go @@ -1,8 +1,8 @@ package config import ( - "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" "go.uber.org/zap" ) @@ -41,7 +41,7 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en return &ConfigDataResponse{ResourceConfig: combinedProperties}, nil } -func getUniqueConfigPropertyList(cMCSNames []chartConfig.CMCSNames, combinedProperties []*ConfigProperty) []*ConfigProperty { +func getUniqueConfigPropertyList(cMCSNames []bean.CMCSNames, combinedProperties []*ConfigProperty) []*ConfigProperty { properties := make([]*ConfigProperty, 0) if len(cMCSNames) == 0 { return properties diff --git a/pkg/pipeline/ConfigMapService.go b/pkg/pipeline/ConfigMapService.go index 0e33e9d8773..15f97949e23 100644 --- a/pkg/pipeline/ConfigMapService.go +++ b/pkg/pipeline/ConfigMapService.go @@ -83,7 +83,7 @@ type ConfigMapService interface { ConfigSecretEnvironmentGet(appId int) ([]bean.JobEnvOverrideResponse, error) ConfigSecretEnvironmentClone(appId int, cloneAppId int, userId int32) ([]chartConfig.ConfigMapEnvModel, error) - FetchCmCsNamesAppAndEnvLevel(appId int, envId int) ([]chartConfig.CMCSNames, []chartConfig.CMCSNames, error) + FetchCmCsNamesAppAndEnvLevel(appId int, envId int) ([]bean.CMCSNames, []bean.CMCSNames, error) } type ConfigMapServiceImpl struct { @@ -1994,8 +1994,8 @@ func (impl ConfigMapServiceImpl) ConfigSecretEnvironmentClone(appId int, cloneAp return jobEnvOverrideResponse, nil } -func (impl ConfigMapServiceImpl) FetchCmCsNamesAppAndEnvLevel(appId int, envId int) ([]chartConfig.CMCSNames, []chartConfig.CMCSNames, error) { - var cMCSNamesEnvLevel []chartConfig.CMCSNames +func (impl ConfigMapServiceImpl) FetchCmCsNamesAppAndEnvLevel(appId int, envId int) ([]bean.CMCSNames, []bean.CMCSNames, error) { + var cMCSNamesEnvLevel []bean.CMCSNames cMCSNamesAppLevel, err := impl.configMapRepository.GetConfigNamesForAppAndEnvLevel(appId, -1) if err != nil { diff --git a/pkg/pipeline/bean/ConfigMapBean.go b/pkg/pipeline/bean/ConfigMapBean.go index ee921eb4613..661294b65b8 100644 --- a/pkg/pipeline/bean/ConfigMapBean.go +++ b/pkg/pipeline/bean/ConfigMapBean.go @@ -91,3 +91,7 @@ type CreateJobEnvOverridePayload struct { type SecretsList struct { ConfigData []*ConfigData `json:"secrets"` } +type CMCSNames struct { + CMName string `json:"cm_name"` + CSName string `json:"cs_name"` +} From 44862e9c9670fd68c31b5b0874a940192c789933 Mon Sep 17 00:00:00 2001 From: adi6859 Date: Wed, 7 Feb 2024 15:07:13 +0530 Subject: [PATCH 17/45] story(configDiffView) : null case handled --- internal/sql/repository/chartConfig/ConfigMapRepository.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index c593e72ce29..55779050f80 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -70,8 +70,8 @@ func (impl ConfigMapRepositoryImpl) GetConfigNamesForAppAndEnvLevel(appId int, e query := impl.dbConnection. Model(). Table(tableName). - ColumnExpr("json_array_elements(config_map_data::json->'maps')->>'name' AS cm_name"). - ColumnExpr("json_array_elements(secret_data::json->'secrets')->>'name' AS cs_name"). + ColumnExpr("json_array_elements(CASE WHEN (config_map_data::json->'maps')::TEXT != 'null' THEN (config_map_data::json->'maps') ELSE '[]' END )->>'name' AS cm_name"). + ColumnExpr("json_array_elements(CASE WHEN (secret_data::json->'secrets')::TEXT != 'null' THEN (secret_data::json->'secrets') ELSE '[]' END )->>'name' AS cs_name"). Where("app_id = ?", appId) if envId > 0 { From 7733ea07586080200f666002d86256e647b5da26 Mon Sep 17 00:00:00 2001 From: adi6859 Date: Wed, 7 Feb 2024 17:23:01 +0530 Subject: [PATCH 18/45] story(configDiffView) : logger added --- pkg/config/DeploymentConfigurationService.go | 1 + pkg/pipeline/ConfigMapService.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/config/DeploymentConfigurationService.go b/pkg/config/DeploymentConfigurationService.go index 9002ff45646..2c616699c98 100644 --- a/pkg/config/DeploymentConfigurationService.go +++ b/pkg/config/DeploymentConfigurationService.go @@ -29,6 +29,7 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, envId int) (*ConfigDataResponse, error) { cMCSNamesAppLevel, cMCSNamesEnvLevel, err := impl.configMapService.FetchCmCsNamesAppAndEnvLevel(appId, envId) if err != nil { + impl.logger.Errorw("error in fetching CM and CS names at app or env level", "appId", appId, "envId", envId) return nil, err } combinedProperties := make([]*ConfigProperty, 0) diff --git a/pkg/pipeline/ConfigMapService.go b/pkg/pipeline/ConfigMapService.go index 15f97949e23..0b19edcacee 100644 --- a/pkg/pipeline/ConfigMapService.go +++ b/pkg/pipeline/ConfigMapService.go @@ -1999,13 +1999,13 @@ func (impl ConfigMapServiceImpl) FetchCmCsNamesAppAndEnvLevel(appId int, envId i cMCSNamesAppLevel, err := impl.configMapRepository.GetConfigNamesForAppAndEnvLevel(appId, -1) if err != nil { - impl.logger.Errorw("error in getting CM/CS names at app level ", "appId", appId, "err", err) + impl.logger.Errorw("error in fetching CM/CS names at app level ", "appId", appId, "err", err) return nil, nil, err } if envId > 0 { cMCSNamesEnvLevel, err = impl.configMapRepository.GetConfigNamesForAppAndEnvLevel(appId, envId) if err != nil { - impl.logger.Errorw("error in getting CM/CS names at env level ", "appId", appId, "envId", envId, "err", err) + impl.logger.Errorw("error in fetching CM/CS names at env level ", "appId", appId, "envId", envId, "err", err) return nil, nil, err } } From e4e39c644debc7b99ca689a4dcd874180eb9a437 Mon Sep 17 00:00:00 2001 From: adi6859 Date: Thu, 8 Feb 2024 11:36:03 +0530 Subject: [PATCH 19/45] story(configDiffView) : code refactored --- api/restHandler/DeploymentConfigurationRestHandler.go | 2 -- .../sql/repository/chartConfig/ConfigMapRepository.go | 9 +++++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 3dc52c7f6ea..d03d6ec6583 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -44,12 +44,10 @@ func (handler *DeploymentConfigurationRestHandlerImpl) ConfigAutoComplete(w http } appId, err := common.ExtractIntQueryParam(w, r, "appId", nil) if err != nil { - common.WriteJsonResp(w, err, "invalid appId", http.StatusBadRequest) return } envId, err := common.ExtractIntQueryParam(w, r, "envId", nil) if err != nil { - common.WriteJsonResp(w, err, "invalid envId", http.StatusBadRequest) return } diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index 55779050f80..e2b9f37d919 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -52,6 +52,11 @@ func NewConfigMapRepositoryImpl(Logger *zap.SugaredLogger, dbConnection *pg.DB) return &ConfigMapRepositoryImpl{dbConnection: dbConnection, Logger: Logger} } +const ( + ConfigMapAppLevel string = "config_map_app_level" + ConfigMapEnvLevel string = "config_map_app_level" +) + type ConfigMapAppModel struct { TableName struct{} `sql:"config_map_app_level" pg:",discard_unknown_columns"` Id int `sql:"id,pk"` @@ -63,9 +68,9 @@ type ConfigMapAppModel struct { func (impl ConfigMapRepositoryImpl) GetConfigNamesForAppAndEnvLevel(appId int, envId int) ([]bean.CMCSNames, error) { var cMCSNames []bean.CMCSNames - tableName := "config_map_env_level" + tableName := ConfigMapEnvLevel if envId == -1 { - tableName = "config_map_app_level" + tableName = ConfigMapAppLevel } query := impl.dbConnection. Model(). From dce967a74064e676c469e75f52879759840c9a6d Mon Sep 17 00:00:00 2001 From: adi6859 Date: Thu, 8 Feb 2024 12:11:12 +0530 Subject: [PATCH 20/45] story(configDiffView) : code refactored v2 --- internal/sql/repository/chartConfig/ConfigMapRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index e2b9f37d919..53ee44d5a8b 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -54,7 +54,7 @@ func NewConfigMapRepositoryImpl(Logger *zap.SugaredLogger, dbConnection *pg.DB) const ( ConfigMapAppLevel string = "config_map_app_level" - ConfigMapEnvLevel string = "config_map_app_level" + ConfigMapEnvLevel string = "config_map_env_level" ) type ConfigMapAppModel struct { From 41741613b7b7b715e5863c5bfc098b0dd1c5728d Mon Sep 17 00:00:00 2001 From: adi6859 Date: Thu, 8 Feb 2024 17:26:31 +0530 Subject: [PATCH 21/45] story(configDiffView) : spec updated --- specs/configDiffView.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/configDiffView.yaml b/specs/configDiffView.yaml index 065fd1ec3f5..2c8b30a483f 100644 --- a/specs/configDiffView.yaml +++ b/specs/configDiffView.yaml @@ -63,8 +63,8 @@ components: ConfigStateEnum: type: integer - enum: [ 1, 2, 3 ] - description: State of config (1 represents published state, 2 represents draft state, 3 represents approval pending state) + enum: [ 3 ] + description: State of config (3 represents published state) ResourceTypeEnum: type: string From 736047a36dbc1176a8135fc56cc852e825d6adb5 Mon Sep 17 00:00:00 2001 From: adi6859 Date: Wed, 14 Feb 2024 12:47:50 +0530 Subject: [PATCH 22/45] story(configDiffView) : code refactored --- .../chartConfig/ConfigMapRepository.go | 29 +++++++++++++++---- pkg/config/DeploymentConfigurationService.go | 23 +++++++-------- pkg/config/bean.go | 20 ++++++------- pkg/pipeline/ConfigMapService.go | 6 ++-- pkg/pipeline/bean/ConfigMapBean.go | 15 ++++++++-- specs/configDiffView.yaml | 6 ++-- 6 files changed, 61 insertions(+), 38 deletions(-) diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index 53ee44d5a8b..35ed3bc29a3 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -40,7 +40,7 @@ type ConfigMapRepository interface { GetByAppIdAppLevel(appId int) (*ConfigMapAppModel, error) GetByAppIdAndEnvIdEnvLevel(appId int, envId int) (*ConfigMapEnvModel, error) GetEnvLevelByAppId(appId int) ([]*ConfigMapEnvModel, error) - GetConfigNamesForAppAndEnvLevel(appId int, envId int) ([]bean.CMCSNames, error) + GetConfigNamesForAppAndEnvLevel(appId int, envId int) ([]bean.ConfigNameAndType, error) } type ConfigMapRepositoryImpl struct { @@ -65,9 +65,13 @@ type ConfigMapAppModel struct { SecretData string `sql:"secret_data"` sql.AuditLog } +type cMCSNames struct { + CMName string `json:"cm_name"` + CSName string `json:"cs_name"` +} -func (impl ConfigMapRepositoryImpl) GetConfigNamesForAppAndEnvLevel(appId int, envId int) ([]bean.CMCSNames, error) { - var cMCSNames []bean.CMCSNames +func (impl ConfigMapRepositoryImpl) GetConfigNamesForAppAndEnvLevel(appId int, envId int) ([]bean.ConfigNameAndType, error) { + var cMCSNames []cMCSNames tableName := ConfigMapEnvLevel if envId == -1 { tableName = ConfigMapAppLevel @@ -85,10 +89,25 @@ func (impl ConfigMapRepositoryImpl) GetConfigNamesForAppAndEnvLevel(appId int, e if err := query.Select(&cMCSNames); err != nil { if err != pg.ErrNoRows { impl.Logger.Errorw("error occurred while fetching CM/CS names", "appId", appId, "err", err) + return nil, err + } + } + var configNameAndType []bean.ConfigNameAndType + for _, name := range cMCSNames { + if name.CMName != "" { + configNameAndType = append(configNameAndType, bean.ConfigNameAndType{ + Name: name.CMName, + Type: bean.CM, + }) + } + if name.CSName != "" { + configNameAndType = append(configNameAndType, bean.ConfigNameAndType{ + Name: name.CSName, + Type: bean.CS, + }) } - return cMCSNames, err } - return cMCSNames, nil + return configNameAndType, nil } func (impl ConfigMapRepositoryImpl) CreateAppLevel(model *ConfigMapAppModel) (*ConfigMapAppModel, error) { diff --git a/pkg/config/DeploymentConfigurationService.go b/pkg/config/DeploymentConfigurationService.go index 2c616699c98..95d8defc7f0 100644 --- a/pkg/config/DeploymentConfigurationService.go +++ b/pkg/config/DeploymentConfigurationService.go @@ -29,7 +29,7 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, envId int) (*ConfigDataResponse, error) { cMCSNamesAppLevel, cMCSNamesEnvLevel, err := impl.configMapService.FetchCmCsNamesAppAndEnvLevel(appId, envId) if err != nil { - impl.logger.Errorw("error in fetching CM and CS names at app or env level", "appId", appId, "envId", envId) + impl.logger.Errorw("error in fetching CM and CS names at app or env level", "appId", appId, "envId", envId, "err", err) return nil, err } combinedProperties := make([]*ConfigProperty, 0) @@ -38,33 +38,30 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en combinedProperties = append(combinedProperties, getUniqueConfigPropertyList(cMCSNamesEnvLevel, combinedProperties)...) combinedProperties = append(combinedProperties, - getConfigProperty("", DeploymentTemplate, PublishedConfigState)) + getConfigProperty("", bean.DeploymentTemplate, PublishedConfigState)) return &ConfigDataResponse{ResourceConfig: combinedProperties}, nil } -func getUniqueConfigPropertyList(cMCSNames []bean.CMCSNames, combinedProperties []*ConfigProperty) []*ConfigProperty { +func getUniqueConfigPropertyList(cMCSNames []bean.ConfigNameAndType, combinedProperties []*ConfigProperty) []*ConfigProperty { properties := make([]*ConfigProperty, 0) if len(cMCSNames) == 0 { return properties } combinedNames := make(map[string]bool) for _, config := range combinedProperties { - combinedNames[config.Name] = true + combinedNames[config.getKey()] = true } - for _, name := range cMCSNames { - // Fill in CM property if the CMName is not empty - if name.CMName != "" && !combinedNames[name.CMName] { - properties = append(properties, getConfigProperty(name.CMName, CM, PublishedConfigState)) - } - // Fill in CS property if the CSName is not empty - if name.CSName != "" && !combinedNames[name.CSName] { - properties = append(properties, getConfigProperty(name.CSName, CS, PublishedConfigState)) + for _, config := range cMCSNames { + // Fill in CM and CS property + property := getConfigProperty(config.Name, config.Type, PublishedConfigState) + if !combinedNames[property.getKey()] { + properties = append(properties, property) } } return properties } -func getConfigProperty(name string, configType ResourceType, State ConfigState) *ConfigProperty { +func getConfigProperty(name string, configType bean.ResourceType, State ConfigState) *ConfigProperty { return &ConfigProperty{ Name: name, Type: configType, diff --git a/pkg/config/bean.go b/pkg/config/bean.go index 6496e22151d..74090413433 100644 --- a/pkg/config/bean.go +++ b/pkg/config/bean.go @@ -1,24 +1,22 @@ package config +import "github.com/devtron-labs/devtron/pkg/pipeline/bean" + type ConfigState int const ( PublishedConfigState ConfigState = 3 ) -type ResourceType string - -const ( - CM ResourceType = "ConfigMap" - CS ResourceType = "Secret" - DeploymentTemplate ResourceType = "Deployment Template" -) - type ConfigProperty struct { - Name string `json:"name"` - ConfigState ConfigState `json:"configState"` - Type ResourceType `json:"type"` + Name string `json:"name"` + ConfigState ConfigState `json:"configState"` + Type bean.ResourceType `json:"type"` } type ConfigDataResponse struct { ResourceConfig []*ConfigProperty `json:"resourceConfig"` } + +func (config ConfigProperty) getKey() string { + return string(config.Type) + config.Name +} diff --git a/pkg/pipeline/ConfigMapService.go b/pkg/pipeline/ConfigMapService.go index 0b19edcacee..09cd35e4b4b 100644 --- a/pkg/pipeline/ConfigMapService.go +++ b/pkg/pipeline/ConfigMapService.go @@ -83,7 +83,7 @@ type ConfigMapService interface { ConfigSecretEnvironmentGet(appId int) ([]bean.JobEnvOverrideResponse, error) ConfigSecretEnvironmentClone(appId int, cloneAppId int, userId int32) ([]chartConfig.ConfigMapEnvModel, error) - FetchCmCsNamesAppAndEnvLevel(appId int, envId int) ([]bean.CMCSNames, []bean.CMCSNames, error) + FetchCmCsNamesAppAndEnvLevel(appId int, envId int) ([]bean.ConfigNameAndType, []bean.ConfigNameAndType, error) } type ConfigMapServiceImpl struct { @@ -1994,8 +1994,8 @@ func (impl ConfigMapServiceImpl) ConfigSecretEnvironmentClone(appId int, cloneAp return jobEnvOverrideResponse, nil } -func (impl ConfigMapServiceImpl) FetchCmCsNamesAppAndEnvLevel(appId int, envId int) ([]bean.CMCSNames, []bean.CMCSNames, error) { - var cMCSNamesEnvLevel []bean.CMCSNames +func (impl ConfigMapServiceImpl) FetchCmCsNamesAppAndEnvLevel(appId int, envId int) ([]bean.ConfigNameAndType, []bean.ConfigNameAndType, error) { + var cMCSNamesEnvLevel []bean.ConfigNameAndType cMCSNamesAppLevel, err := impl.configMapRepository.GetConfigNamesForAppAndEnvLevel(appId, -1) if err != nil { diff --git a/pkg/pipeline/bean/ConfigMapBean.go b/pkg/pipeline/bean/ConfigMapBean.go index 661294b65b8..b568f7a07fa 100644 --- a/pkg/pipeline/bean/ConfigMapBean.go +++ b/pkg/pipeline/bean/ConfigMapBean.go @@ -91,7 +91,16 @@ type CreateJobEnvOverridePayload struct { type SecretsList struct { ConfigData []*ConfigData `json:"secrets"` } -type CMCSNames struct { - CMName string `json:"cm_name"` - CSName string `json:"cs_name"` + +type ConfigNameAndType struct { + Name string + Type ResourceType } + +type ResourceType string + +const ( + CM ResourceType = "ConfigMap" + CS ResourceType = "Secret" + DeploymentTemplate ResourceType = "Deployment Template" +) diff --git a/specs/configDiffView.yaml b/specs/configDiffView.yaml index 2c8b30a483f..8a24d50989c 100644 --- a/specs/configDiffView.yaml +++ b/specs/configDiffView.yaml @@ -62,9 +62,9 @@ components: $ref: '#/components/schemas/ResourceTypeEnum' ConfigStateEnum: - type: integer - enum: [ 3 ] - description: State of config (3 represents published state) + type: integer + enum: [ 1, 2, 3 ] + description: State of config (1 represents draft state , 2 represents approval pending state,3 represents published state) ResourceTypeEnum: type: string From 16d07d376c5a542da4467fbf0193e70a75dfc441 Mon Sep 17 00:00:00 2001 From: adi6859 Date: Wed, 14 Feb 2024 13:01:39 +0530 Subject: [PATCH 23/45] story(configDiffView) : config names --- .../sql/repository/chartConfig/ConfigMapRepository.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index 35ed3bc29a3..78f51c4119d 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -92,22 +92,22 @@ func (impl ConfigMapRepositoryImpl) GetConfigNamesForAppAndEnvLevel(appId int, e return nil, err } } - var configNameAndType []bean.ConfigNameAndType + var configNames []bean.ConfigNameAndType for _, name := range cMCSNames { if name.CMName != "" { - configNameAndType = append(configNameAndType, bean.ConfigNameAndType{ + configNames = append(configNames, bean.ConfigNameAndType{ Name: name.CMName, Type: bean.CM, }) } if name.CSName != "" { - configNameAndType = append(configNameAndType, bean.ConfigNameAndType{ + configNames = append(configNames, bean.ConfigNameAndType{ Name: name.CSName, Type: bean.CS, }) } } - return configNameAndType, nil + return configNames, nil } func (impl ConfigMapRepositoryImpl) CreateAppLevel(model *ConfigMapAppModel) (*ConfigMapAppModel, error) { From 6196f315e2208215c7c229eca89ae29a02a257e2 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 3 Jul 2024 11:14:18 +0530 Subject: [PATCH 24/45] main sync --- env_gen.md | 2 +- wire_gen.go | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/env_gen.md b/env_gen.md index 1ed120c5f6e..a91e8b6655b 100644 --- a/env_gen.md +++ b/env_gen.md @@ -245,8 +245,8 @@ | USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW | true | | | USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW | true | | | USE_BUILDX | false | | - | USE_CUSTOM_HTTP_TRANSPORT | false | | | USE_CASBIN_V2 | false | | + | USE_CUSTOM_HTTP_TRANSPORT | false | | | USE_EXTERNAL_NODE | false | | | USE_GIT_CLI | false | | | USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD | false | | diff --git a/wire_gen.go b/wire_gen.go index 2d0e72a2b2f..d44cfda5696 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -130,6 +130,7 @@ import ( "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess" "github.com/devtron-labs/devtron/pkg/commonService" + config2 "github.com/devtron-labs/devtron/pkg/config" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" @@ -307,7 +308,11 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - enforcerImpl, err := casbin.NewEnforcerImpl(syncedEnforcer, sessionManager, sugaredLogger) + casbinSyncedEnforcer, err := casbin.CreateV2() + if err != nil { + return nil, err + } + enforcerImpl, err := casbin.NewEnforcerImpl(syncedEnforcer, casbinSyncedEnforcer, sessionManager, sugaredLogger) if err != nil { return nil, err } @@ -918,6 +923,12 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } + deploymentConfigurationServiceImpl, err := config2.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl) + if err != nil { + return nil, err + } + deploymentConfigurationRestHandlerImpl := restHandler.NewDeploymentConfigurationRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerUtilImpl, deploymentConfigurationServiceImpl) + deploymentConfigurationRouterImpl := router.NewDeploymentConfigurationRouter(deploymentConfigurationRestHandlerImpl) infraConfigRestHandlerImpl := infraConfig2.NewInfraConfigRestHandlerImpl(sugaredLogger, infraConfigServiceImpl, userServiceImpl, enforcerImpl, enforcerUtilImpl, validate) infraConfigRouterImpl := infraConfig2.NewInfraProfileRouterImpl(infraConfigRestHandlerImpl) argoApplicationRestHandlerImpl := argoApplication2.NewArgoApplicationRestHandlerImpl(argoApplicationServiceImpl, sugaredLogger, enforcerImpl) @@ -927,7 +938,7 @@ func InitializeApp() (*App, error) { historyRestHandlerImpl := devtronResource2.NewHistoryRestHandlerImpl(sugaredLogger, enforcerImpl, deploymentHistoryServiceImpl, apiReqDecoderServiceImpl, enforcerUtilImpl) historyRouterImpl := devtronResource2.NewHistoryRouterImpl(historyRestHandlerImpl) devtronResourceRouterImpl := devtronResource2.NewDevtronResourceRouterImpl(historyRouterImpl) - muxRouter := router.NewMuxRouter(sugaredLogger, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl, infraConfigRouterImpl, argoApplicationRouterImpl, devtronResourceRouterImpl) + muxRouter := router.NewMuxRouter(sugaredLogger, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl, deploymentConfigurationRouterImpl, infraConfigRouterImpl, argoApplicationRouterImpl, devtronResourceRouterImpl) loggingMiddlewareImpl := util4.NewLoggingMiddlewareImpl(userServiceImpl) cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) @@ -944,6 +955,6 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, syncedEnforcer, db, sessionManager, posthogClient, loggingMiddlewareImpl, centralEventProcessor, pubSubClientServiceImpl, workflowEventProcessorImpl) + mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, syncedEnforcer, db, sessionManager, posthogClient, loggingMiddlewareImpl, centralEventProcessor, pubSubClientServiceImpl, workflowEventProcessorImpl, casbinSyncedEnforcer) return mainApp, nil } From e076783106b1ba7da45fc98f26a32606cae69794 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 3 Jul 2024 17:26:35 +0530 Subject: [PATCH 25/45] overridden and global flag introduced in config diff autocomplete api --- Wire.go | 6 +- .../DeploymentConfigurationRestHandler.go | 10 +-- .../DeploymentPipelineRestHandler.go | 2 +- api/restHandler/common/ParamParserUtils.go | 6 +- api/router/DeploymentConfigRouter.go | 2 - pkg/config/DeploymentConfigurationService.go | 70 ------------------- pkg/config/bean.go | 22 ------ .../DeploymentConfigurationService.go | 51 ++++++++++++++ pkg/configDiff/bean/bean.go | 45 ++++++++++++ pkg/configDiff/helper/helper.go | 67 ++++++++++++++++++ wire_gen.go | 4 +- 11 files changed, 177 insertions(+), 108 deletions(-) delete mode 100644 pkg/config/DeploymentConfigurationService.go delete mode 100644 pkg/config/bean.go create mode 100644 pkg/configDiff/DeploymentConfigurationService.go create mode 100644 pkg/configDiff/bean/bean.go create mode 100644 pkg/configDiff/helper/helper.go diff --git a/Wire.go b/Wire.go index 62732b0addd..3e9cc222f5c 100644 --- a/Wire.go +++ b/Wire.go @@ -119,7 +119,7 @@ import ( "github.com/devtron-labs/devtron/pkg/chart/gitOpsConfig" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/commonService" - "github.com/devtron-labs/devtron/pkg/config" + "github.com/devtron-labs/devtron/pkg/configDiff" delete2 "github.com/devtron-labs/devtron/pkg/delete" deployment2 "github.com/devtron-labs/devtron/pkg/deployment" git2 "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" @@ -704,8 +704,8 @@ func InitializeApp() (*App, error) { wire.Bind(new(router.DeploymentConfigurationRouter), new(*router.DeploymentConfigurationRouterImpl)), restHandler.NewDeploymentConfigurationRestHandlerImpl, wire.Bind(new(restHandler.DeploymentConfigurationRestHandler), new(*restHandler.DeploymentConfigurationRestHandlerImpl)), - config.NewDeploymentConfigurationServiceImpl, - wire.Bind(new(config.DeploymentConfigurationService), new(*config.DeploymentConfigurationServiceImpl)), + configDiff.NewDeploymentConfigurationServiceImpl, + wire.Bind(new(configDiff.DeploymentConfigurationService), new(*configDiff.DeploymentConfigurationServiceImpl)), router.NewTelemetryRouterImpl, wire.Bind(new(router.TelemetryRouter), new(*router.TelemetryRouterImpl)), diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index d03d6ec6583..567ff9895ba 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -5,7 +5,7 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/config" + "github.com/devtron-labs/devtron/pkg/configDiff" "github.com/devtron-labs/devtron/util/rbac" "go.uber.org/zap" "gopkg.in/go-playground/validator.v9" @@ -20,13 +20,13 @@ type DeploymentConfigurationRestHandlerImpl struct { userAuthService user.UserService validator *validator.Validate enforcerUtil rbac.EnforcerUtil - deploymentConfigurationService config.DeploymentConfigurationService + deploymentConfigurationService configDiff.DeploymentConfigurationService } func NewDeploymentConfigurationRestHandlerImpl(logger *zap.SugaredLogger, userAuthService user.UserService, enforcerUtil rbac.EnforcerUtil, - deploymentConfigurationService config.DeploymentConfigurationService, + deploymentConfigurationService configDiff.DeploymentConfigurationService, ) *DeploymentConfigurationRestHandlerImpl { return &DeploymentConfigurationRestHandlerImpl{ logger: logger, @@ -42,11 +42,11 @@ func (handler *DeploymentConfigurationRestHandlerImpl) ConfigAutoComplete(w http common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - appId, err := common.ExtractIntQueryParam(w, r, "appId", nil) + appId, err := common.ExtractIntQueryParam(w, r, "appId", 0) if err != nil { return } - envId, err := common.ExtractIntQueryParam(w, r, "envId", nil) + envId, err := common.ExtractIntQueryParam(w, r, "envId", 0) if err != nil { return } diff --git a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go index 33a60219b83..f3b880fe516 100644 --- a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go @@ -984,7 +984,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetRestartWorkloadData(w http.Resp common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - envId, err := common.ExtractIntQueryParam(w, r, "envId", nil) + envId, err := common.ExtractIntQueryParam(w, r, "envId", 0) if err != nil { return } diff --git a/api/restHandler/common/ParamParserUtils.go b/api/restHandler/common/ParamParserUtils.go index ec5d849bdb9..8ba342285d4 100644 --- a/api/restHandler/common/ParamParserUtils.go +++ b/api/restHandler/common/ParamParserUtils.go @@ -58,15 +58,15 @@ func convertToIntArray(w http.ResponseWriter, paramValue string) ([]int, error) return paramValues, nil } -func ExtractIntQueryParam(w http.ResponseWriter, r *http.Request, paramName string, defaultVal *int) (int, error) { +func ExtractIntQueryParam(w http.ResponseWriter, r *http.Request, paramName string, defaultVal int) (int, error) { queryParams := r.URL.Query() paramValue := queryParams.Get(paramName) if len(paramValue) == 0 { - return *defaultVal, nil + return defaultVal, nil } paramIntValue, err := convertToInt(w, paramValue) if err != nil { - return 0, err + return defaultVal, err } return paramIntValue, nil } diff --git a/api/router/DeploymentConfigRouter.go b/api/router/DeploymentConfigRouter.go index 6792e0c1fe3..941118099a4 100644 --- a/api/router/DeploymentConfigRouter.go +++ b/api/router/DeploymentConfigRouter.go @@ -23,7 +23,5 @@ func NewDeploymentConfigurationRouter(deploymentGroupRestHandler restHandler.Dep func (router DeploymentConfigurationRouterImpl) initDeploymentConfigurationRouter(configRouter *mux.Router) { configRouter.Path("/autocomplete"). HandlerFunc(router.deploymentGroupRestHandler.ConfigAutoComplete). - Queries("appId", "{appId}"). - Queries("envId", "{envId}"). Methods("GET") } diff --git a/pkg/config/DeploymentConfigurationService.go b/pkg/config/DeploymentConfigurationService.go deleted file mode 100644 index 95d8defc7f0..00000000000 --- a/pkg/config/DeploymentConfigurationService.go +++ /dev/null @@ -1,70 +0,0 @@ -package config - -import ( - "github.com/devtron-labs/devtron/pkg/pipeline" - "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "go.uber.org/zap" -) - -type DeploymentConfigurationService interface { - ConfigAutoComplete(appId int, envId int) (*ConfigDataResponse, error) -} - -type DeploymentConfigurationServiceImpl struct { - logger *zap.SugaredLogger - configMapService pipeline.ConfigMapService -} - -func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, - configMapService pipeline.ConfigMapService, -) (*DeploymentConfigurationServiceImpl, error) { - deploymentConfigurationService := &DeploymentConfigurationServiceImpl{ - logger: logger, - configMapService: configMapService, - } - - return deploymentConfigurationService, nil -} - -func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, envId int) (*ConfigDataResponse, error) { - cMCSNamesAppLevel, cMCSNamesEnvLevel, err := impl.configMapService.FetchCmCsNamesAppAndEnvLevel(appId, envId) - if err != nil { - impl.logger.Errorw("error in fetching CM and CS names at app or env level", "appId", appId, "envId", envId, "err", err) - return nil, err - } - combinedProperties := make([]*ConfigProperty, 0) - combinedProperties = append(combinedProperties, - getUniqueConfigPropertyList(cMCSNamesAppLevel, combinedProperties)...) - combinedProperties = append(combinedProperties, - getUniqueConfigPropertyList(cMCSNamesEnvLevel, combinedProperties)...) - combinedProperties = append(combinedProperties, - getConfigProperty("", bean.DeploymentTemplate, PublishedConfigState)) - return &ConfigDataResponse{ResourceConfig: combinedProperties}, nil -} - -func getUniqueConfigPropertyList(cMCSNames []bean.ConfigNameAndType, combinedProperties []*ConfigProperty) []*ConfigProperty { - properties := make([]*ConfigProperty, 0) - if len(cMCSNames) == 0 { - return properties - } - combinedNames := make(map[string]bool) - for _, config := range combinedProperties { - combinedNames[config.getKey()] = true - } - for _, config := range cMCSNames { - // Fill in CM and CS property - property := getConfigProperty(config.Name, config.Type, PublishedConfigState) - if !combinedNames[property.getKey()] { - properties = append(properties, property) - } - } - return properties -} - -func getConfigProperty(name string, configType bean.ResourceType, State ConfigState) *ConfigProperty { - return &ConfigProperty{ - Name: name, - Type: configType, - ConfigState: State, - } -} diff --git a/pkg/config/bean.go b/pkg/config/bean.go deleted file mode 100644 index 74090413433..00000000000 --- a/pkg/config/bean.go +++ /dev/null @@ -1,22 +0,0 @@ -package config - -import "github.com/devtron-labs/devtron/pkg/pipeline/bean" - -type ConfigState int - -const ( - PublishedConfigState ConfigState = 3 -) - -type ConfigProperty struct { - Name string `json:"name"` - ConfigState ConfigState `json:"configState"` - Type bean.ResourceType `json:"type"` -} -type ConfigDataResponse struct { - ResourceConfig []*ConfigProperty `json:"resourceConfig"` -} - -func (config ConfigProperty) getKey() string { - return string(config.Type) + config.Name -} diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go new file mode 100644 index 00000000000..b349e27b2b2 --- /dev/null +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -0,0 +1,51 @@ +package configDiff + +import ( + bean2 "github.com/devtron-labs/devtron/pkg/configDiff/bean" + "github.com/devtron-labs/devtron/pkg/configDiff/helper" + "github.com/devtron-labs/devtron/pkg/pipeline" + "go.uber.org/zap" +) + +type DeploymentConfigurationService interface { + ConfigAutoComplete(appId int, envId int) (*bean2.ConfigDataResponse, error) +} + +type DeploymentConfigurationServiceImpl struct { + logger *zap.SugaredLogger + configMapService pipeline.ConfigMapService +} + +func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, + configMapService pipeline.ConfigMapService, +) (*DeploymentConfigurationServiceImpl, error) { + deploymentConfigurationService := &DeploymentConfigurationServiceImpl{ + logger: logger, + configMapService: configMapService, + } + + return deploymentConfigurationService, nil +} + +func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, envId int) (*bean2.ConfigDataResponse, error) { + cMCSNamesAppLevel, cMCSNamesEnvLevel, err := impl.configMapService.FetchCmCsNamesAppAndEnvLevel(appId, envId) + if err != nil { + impl.logger.Errorw("error in fetching CM and CS names at app or env level", "appId", appId, "envId", envId, "err", err) + return nil, err + } + cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap := helper.GetCmCsAppAndEnvLevelMap(cMCSNamesAppLevel, cMCSNamesEnvLevel) + for key, configProperty := range cmcsKeyPropertyAppLevelMap { + if _, ok := cmcsKeyPropertyEnvLevelMap[key]; !ok { + configProperty.Global = true + } + } + for key, configProperty := range cmcsKeyPropertyEnvLevelMap { + if _, ok := cmcsKeyPropertyAppLevelMap[key]; ok { + configProperty.Global = true + configProperty.Overridden = true + } + } + combinedProperties := helper.GetCombinedPropertiesMap(cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap) + + return &bean2.ConfigDataResponse{ResourceConfig: combinedProperties}, nil +} diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go new file mode 100644 index 00000000000..c802ac97a84 --- /dev/null +++ b/pkg/configDiff/bean/bean.go @@ -0,0 +1,45 @@ +package bean + +import ( + "fmt" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" +) + +type ConfigState int + +const ( + PublishedConfigState ConfigState = 3 +) + +type ConfigProperty struct { + Name string `json:"name"` + ConfigState ConfigState `json:"configState"` + Type bean.ResourceType `json:"type"` + Overridden bool `json:"overridden"` + Global bool `json:"global"` +} + +func NewConfigProperty() *ConfigProperty { + return &ConfigProperty{} +} + +func (r *ConfigProperty) IsConfigPropertyGlobal() bool { + return r.Global +} + +func (r *ConfigProperty) SetConfigProperty(Name string, ConfigState ConfigState, Type bean.ResourceType, Overridden bool, Global bool) *ConfigProperty { + r.Name = Name + r.ConfigState = ConfigState + r.Type = Type + r.Overridden = Overridden + r.Global = Global + return r +} + +type ConfigDataResponse struct { + ResourceConfig []*ConfigProperty `json:"resourceConfig"` +} + +func (r *ConfigProperty) GetKey() string { + return fmt.Sprintf("%s-%s", string(r.Type), r.Name) +} diff --git a/pkg/configDiff/helper/helper.go b/pkg/configDiff/helper/helper.go new file mode 100644 index 00000000000..e56daaf5f41 --- /dev/null +++ b/pkg/configDiff/helper/helper.go @@ -0,0 +1,67 @@ +package helper + +import ( + bean2 "github.com/devtron-labs/devtron/pkg/configDiff/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" +) + +func GetCmCsAppAndEnvLevelMap(cMCSNamesAppLevel, cMCSNamesEnvLevel []bean.ConfigNameAndType) (map[string]*bean2.ConfigProperty, map[string]*bean2.ConfigProperty) { + cMCSNamesAppLevelMap, cMCSNamesEnvLevelMap := make(map[string]*bean2.ConfigProperty, len(cMCSNamesAppLevel)), make(map[string]*bean2.ConfigProperty, len(cMCSNamesEnvLevel)) + + for _, cmcs := range cMCSNamesAppLevel { + property := GetConfigProperty(cmcs.Name, cmcs.Type, bean2.PublishedConfigState) + cMCSNamesAppLevelMap[property.GetKey()] = property + } + for _, cmcs := range cMCSNamesEnvLevel { + property := GetConfigProperty(cmcs.Name, cmcs.Type, bean2.PublishedConfigState) + cMCSNamesEnvLevelMap[property.GetKey()] = property + } + return cMCSNamesAppLevelMap, cMCSNamesEnvLevelMap +} + +func GetConfigProperty(name string, configType bean.ResourceType, State bean2.ConfigState) *bean2.ConfigProperty { + return &bean2.ConfigProperty{ + Name: name, + Type: configType, + ConfigState: State, + } +} + +//func GetUniqueConfigPropertyList(cmcsKeyVsPropertyMap map[string]*bean2.ConfigProperty, combinedProperties []*bean2.ConfigProperty) []*bean2.ConfigProperty { +// properties := make([]*bean2.ConfigProperty, 0) +// if len(cmcsKeyVsPropertyMap) == 0 { +// return properties +// } +// +// combinedNames := make(map[string]bool) +// for _, config := range combinedProperties { +// combinedNames[config.GetKey()] = true +// } +// +// for key, property := range cmcsKeyVsPropertyMap { +// if !combinedNames[key] { +// properties = append(properties, property) +// } else { +// cmcsKeyVsPropertyMap[key] = +// } +// } +// return properties +//} + +func GetCombinedPropertiesMap(cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap map[string]*bean2.ConfigProperty) []*bean2.ConfigProperty { + combinedPropertiesMap := make(map[string]*bean2.ConfigProperty, len(cmcsKeyPropertyAppLevelMap)+len(cmcsKeyPropertyEnvLevelMap)) + for key, property := range cmcsKeyPropertyAppLevelMap { + if property.IsConfigPropertyGlobal() { + // only append global =true in combined for app level cmcs, all overridden cmcs would be handled in cmcsKeyPropertyEnvLevelMap + combinedPropertiesMap[key] = property + } + } + for key, property := range cmcsKeyPropertyEnvLevelMap { + combinedPropertiesMap[key] = property + } + combinedProperties := make([]*bean2.ConfigProperty, 0, len(cmcsKeyPropertyAppLevelMap)+len(cmcsKeyPropertyEnvLevelMap)) + for _, property := range combinedPropertiesMap { + combinedProperties = append(combinedProperties, property) + } + return combinedProperties +} diff --git a/wire_gen.go b/wire_gen.go index d44cfda5696..1efab2de1ac 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -130,7 +130,7 @@ import ( "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess" "github.com/devtron-labs/devtron/pkg/commonService" - config2 "github.com/devtron-labs/devtron/pkg/config" + "github.com/devtron-labs/devtron/pkg/configDiff" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" @@ -923,7 +923,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - deploymentConfigurationServiceImpl, err := config2.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl) + deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl) if err != nil { return nil, err } From ffa3ba1441ad1745f9f6fcf9ed42baa35992aabf Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 9 Jul 2024 19:34:07 +0530 Subject: [PATCH 26/45] ent sync --- .../chartConfig/ConfigMapRepository.go | 4 +++ .../DeploymentConfigurationService.go | 6 ++-- pkg/configDiff/bean/bean.go | 22 +++++++++++++++ pkg/configDiff/helper/helper.go | 28 +++---------------- pkg/pipeline/bean/ConfigMapBean.go | 1 + 5 files changed, 35 insertions(+), 26 deletions(-) diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index 513a9baff6e..67f2edec141 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -65,6 +65,7 @@ type ConfigMapAppModel struct { sql.AuditLog } type cMCSNames struct { + Id int `json:"id"` CMName string `json:"cm_name"` CSName string `json:"cs_name"` } @@ -78,6 +79,7 @@ func (impl ConfigMapRepositoryImpl) GetConfigNamesForAppAndEnvLevel(appId int, e query := impl.dbConnection. Model(). Table(tableName). + Column("id"). ColumnExpr("json_array_elements(CASE WHEN (config_map_data::json->'maps')::TEXT != 'null' THEN (config_map_data::json->'maps') ELSE '[]' END )->>'name' AS cm_name"). ColumnExpr("json_array_elements(CASE WHEN (secret_data::json->'secrets')::TEXT != 'null' THEN (secret_data::json->'secrets') ELSE '[]' END )->>'name' AS cs_name"). Where("app_id = ?", appId) @@ -95,12 +97,14 @@ func (impl ConfigMapRepositoryImpl) GetConfigNamesForAppAndEnvLevel(appId int, e for _, name := range cMCSNames { if name.CMName != "" { configNames = append(configNames, bean.ConfigNameAndType{ + Id: name.Id, Name: name.CMName, Type: bean.CM, }) } if name.CSName != "" { configNames = append(configNames, bean.ConfigNameAndType{ + Id: name.Id, Name: name.CSName, Type: bean.CS, }) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index b349e27b2b2..2df817b80b1 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -4,6 +4,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/configDiff/bean" "github.com/devtron-labs/devtron/pkg/configDiff/helper" "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" "go.uber.org/zap" ) @@ -26,7 +27,6 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, return deploymentConfigurationService, nil } - func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, envId int) (*bean2.ConfigDataResponse, error) { cMCSNamesAppLevel, cMCSNamesEnvLevel, err := impl.configMapService.FetchCmCsNamesAppAndEnvLevel(appId, envId) if err != nil { @@ -46,6 +46,8 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en } } combinedProperties := helper.GetCombinedPropertiesMap(cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap) + combinedProperties = append(combinedProperties, helper.GetConfigProperty(0, "", bean.DeploymentTemplate, bean2.PublishedConfigState)) - return &bean2.ConfigDataResponse{ResourceConfig: combinedProperties}, nil + configDataResp := bean2.NewConfigDataResponse().WithResourceConfig(combinedProperties) + return configDataResp, nil } diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index c802ac97a84..c7f1d538ea9 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -12,6 +12,7 @@ const ( ) type ConfigProperty struct { + Id int `json:"id"` Name string `json:"name"` ConfigState ConfigState `json:"configState"` Type bean.ResourceType `json:"type"` @@ -40,6 +41,27 @@ type ConfigDataResponse struct { ResourceConfig []*ConfigProperty `json:"resourceConfig"` } +func NewConfigDataResponse() *ConfigDataResponse { + return &ConfigDataResponse{} +} + +func (r *ConfigDataResponse) WithResourceConfig(resourceConfig []*ConfigProperty) *ConfigDataResponse { + r.ResourceConfig = resourceConfig + return r +} + func (r *ConfigProperty) GetKey() string { return fmt.Sprintf("%s-%s", string(r.Type), r.Name) } + +type ConfigPropertyIdentifier struct { + Name string `json:"name"` + Type bean.ResourceType `json:"type"` +} + +func (r *ConfigProperty) GetIdentifier() ConfigPropertyIdentifier { + return ConfigPropertyIdentifier{ + Name: r.Name, + Type: r.Type, + } +} diff --git a/pkg/configDiff/helper/helper.go b/pkg/configDiff/helper/helper.go index e56daaf5f41..3182cc16d9a 100644 --- a/pkg/configDiff/helper/helper.go +++ b/pkg/configDiff/helper/helper.go @@ -9,45 +9,25 @@ func GetCmCsAppAndEnvLevelMap(cMCSNamesAppLevel, cMCSNamesEnvLevel []bean.Config cMCSNamesAppLevelMap, cMCSNamesEnvLevelMap := make(map[string]*bean2.ConfigProperty, len(cMCSNamesAppLevel)), make(map[string]*bean2.ConfigProperty, len(cMCSNamesEnvLevel)) for _, cmcs := range cMCSNamesAppLevel { - property := GetConfigProperty(cmcs.Name, cmcs.Type, bean2.PublishedConfigState) + property := GetConfigProperty(cmcs.Id, cmcs.Name, cmcs.Type, bean2.PublishedConfigState) cMCSNamesAppLevelMap[property.GetKey()] = property } for _, cmcs := range cMCSNamesEnvLevel { - property := GetConfigProperty(cmcs.Name, cmcs.Type, bean2.PublishedConfigState) + property := GetConfigProperty(cmcs.Id, cmcs.Name, cmcs.Type, bean2.PublishedConfigState) cMCSNamesEnvLevelMap[property.GetKey()] = property } return cMCSNamesAppLevelMap, cMCSNamesEnvLevelMap } -func GetConfigProperty(name string, configType bean.ResourceType, State bean2.ConfigState) *bean2.ConfigProperty { +func GetConfigProperty(id int, name string, configType bean.ResourceType, State bean2.ConfigState) *bean2.ConfigProperty { return &bean2.ConfigProperty{ + Id: id, Name: name, Type: configType, ConfigState: State, } } -//func GetUniqueConfigPropertyList(cmcsKeyVsPropertyMap map[string]*bean2.ConfigProperty, combinedProperties []*bean2.ConfigProperty) []*bean2.ConfigProperty { -// properties := make([]*bean2.ConfigProperty, 0) -// if len(cmcsKeyVsPropertyMap) == 0 { -// return properties -// } -// -// combinedNames := make(map[string]bool) -// for _, config := range combinedProperties { -// combinedNames[config.GetKey()] = true -// } -// -// for key, property := range cmcsKeyVsPropertyMap { -// if !combinedNames[key] { -// properties = append(properties, property) -// } else { -// cmcsKeyVsPropertyMap[key] = -// } -// } -// return properties -//} - func GetCombinedPropertiesMap(cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap map[string]*bean2.ConfigProperty) []*bean2.ConfigProperty { combinedPropertiesMap := make(map[string]*bean2.ConfigProperty, len(cmcsKeyPropertyAppLevelMap)+len(cmcsKeyPropertyEnvLevelMap)) for key, property := range cmcsKeyPropertyAppLevelMap { diff --git a/pkg/pipeline/bean/ConfigMapBean.go b/pkg/pipeline/bean/ConfigMapBean.go index 351b27a34a0..e66fc8dfa03 100644 --- a/pkg/pipeline/bean/ConfigMapBean.go +++ b/pkg/pipeline/bean/ConfigMapBean.go @@ -109,6 +109,7 @@ type SecretsList struct { } type ConfigNameAndType struct { + Id int Name string Type ResourceType } From 7968e750933976d4cb2edc83bcb87764107022e4 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 10 Jul 2024 14:31:07 +0530 Subject: [PATCH 27/45] get config data in resthandler --- .../DeploymentConfigurationRestHandler.go | 36 +++++++++++++++++++ api/router/DeploymentConfigRouter.go | 4 +++ pkg/configDiff/bean/bean.go | 14 ++++++++ 3 files changed, 54 insertions(+) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 567ff9895ba..cab30078d64 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -14,6 +14,7 @@ import ( type DeploymentConfigurationRestHandler interface { ConfigAutoComplete(w http.ResponseWriter, r *http.Request) + GetConfigData(w http.ResponseWriter, r *http.Request) } type DeploymentConfigurationRestHandlerImpl struct { logger *zap.SugaredLogger @@ -69,3 +70,38 @@ func (handler *DeploymentConfigurationRestHandlerImpl) ConfigAutoComplete(w http } common.WriteJsonResp(w, err, res, http.StatusOK) } + +func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.ResponseWriter, r *http.Request) { + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + + appId, err := common.ExtractIntQueryParam(w, r, "appId", 0) + if err != nil { + return + } + envId, err := common.ExtractIntQueryParam(w, r, "envId", 0) + if err != nil { + return + } + + //RBAC START + token := r.Header.Get(common.TokenHeaderKey) + object := handler.enforcerUtil.GetAppRBACNameByAppId(appId) + ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, casbin.ActionGet) + if !ok { + common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), nil, http.StatusForbidden) + return + } + //RBAC END + + res, err := handler.deploymentConfigurationService.ConfigAutoComplete(appId, envId) + if err != nil { + handler.logger.Errorw("service err, ConfigAutoComplete ", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, err, res, http.StatusOK) +} diff --git a/api/router/DeploymentConfigRouter.go b/api/router/DeploymentConfigRouter.go index 941118099a4..a8a568d6046 100644 --- a/api/router/DeploymentConfigRouter.go +++ b/api/router/DeploymentConfigRouter.go @@ -24,4 +24,8 @@ func (router DeploymentConfigurationRouterImpl) initDeploymentConfigurationRoute configRouter.Path("/autocomplete"). HandlerFunc(router.deploymentGroupRestHandler.ConfigAutoComplete). Methods("GET") + configRouter.Path("/data"). + HandlerFunc(router.deploymentGroupRestHandler.GetConfigData). + Methods("GET") + } diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index c7f1d538ea9..1a6eedd62bc 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -65,3 +65,17 @@ func (r *ConfigProperty) GetIdentifier() ConfigPropertyIdentifier { Type: r.Type, } } + +type DeploymentAndCmCsConfig struct { + Id int `json:"id ,omitempty"` + Name string `json:"name"` + CmCsType string `json:"cmcsType ,omitempty"` //example:- environment or volume, in case of cm cs + Data string `json:"data"` + ConfigState ConfigState `json:"configState"` +} + +type DeploymentAndCmCsResponseDto struct { + AppId int `json:"appId"` + EnvironmentId int `json:"environmentId"` + ConfigData []*DeploymentAndCmCsConfig `json:"configData"` +} From c85395fa50800e0ea4d00f483dea758c5e04f455 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 11 Jul 2024 18:27:09 +0530 Subject: [PATCH 28/45] new api for showing all config data in config/data :- Service func -> GetAllConfigData --- .../DeploymentConfigurationRestHandler.go | 58 ++++-- .../DeploymentConfigurationService.go | 182 +++++++++++++++++- pkg/configDiff/bean/bean.go | 103 +++++++++- pkg/configDiff/utils/utils.go | 16 ++ pkg/pipeline/ConfigMapService.go | 32 +++ wire_gen.go | 2 +- 6 files changed, 363 insertions(+), 30 deletions(-) create mode 100644 pkg/configDiff/utils/utils.go diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index cab30078d64..c59f45212a1 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -6,10 +6,12 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/configDiff" + "github.com/devtron-labs/devtron/pkg/configDiff/bean" "github.com/devtron-labs/devtron/util/rbac" "go.uber.org/zap" "gopkg.in/go-playground/validator.v9" "net/http" + "strconv" ) type DeploymentConfigurationRestHandler interface { @@ -77,31 +79,53 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - - appId, err := common.ExtractIntQueryParam(w, r, "appId", 0) - if err != nil { - return - } - envId, err := common.ExtractIntQueryParam(w, r, "envId", 0) + configDataQueryParams, err := getConfigDataQueryParams(r) if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - //RBAC START - token := r.Header.Get(common.TokenHeaderKey) - object := handler.enforcerUtil.GetAppRBACNameByAppId(appId) - ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, casbin.ActionGet) - if !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), nil, http.StatusForbidden) - return + if len(configDataQueryParams.AppName) > 0 { + //RBAC START + token := r.Header.Get(common.TokenHeaderKey) + object := handler.enforcerUtil.GetAppRBACName(configDataQueryParams.AppName) + ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, casbin.ActionGet) + if !ok { + common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), nil, http.StatusForbidden) + return + } + //RBAC END } - //RBAC END - res, err := handler.deploymentConfigurationService.ConfigAutoComplete(appId, envId) + res, err := handler.deploymentConfigurationService.GetAllConfigData(r.Context(), configDataQueryParams) if err != nil { - handler.logger.Errorw("service err, ConfigAutoComplete ", "err", err) + handler.logger.Errorw("service err, GetAllConfigData ", "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - common.WriteJsonResp(w, err, res, http.StatusOK) + common.WriteJsonResp(w, nil, res, http.StatusOK) +} + +func getConfigDataQueryParams(r *http.Request) (*bean.ConfigDataQueryParams, error) { + v := r.URL.Query() + appName := v.Get("appName") + envName := v.Get("envName") + configType := v.Get("configType") + identifierIdStr := v.Get("identifierId") + + identifierId, err := strconv.Atoi(identifierIdStr) + if err != nil { + return nil, err + } + + resourceName := v.Get("resourceName") + resourceType := v.Get("resourceType") + return &bean.ConfigDataQueryParams{ + AppName: appName, + EnvName: envName, + ConfigType: configType, + IdentifierId: identifierId, + ResourceName: resourceName, + ResourceType: resourceType, + }, nil } diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 2df817b80b1..0ad0c1cdfcb 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -1,28 +1,52 @@ package configDiff import ( + "context" + "encoding/json" + repository2 "github.com/devtron-labs/devtron/internal/sql/repository" + appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/util" + chartService "github.com/devtron-labs/devtron/pkg/chart" + "github.com/devtron-labs/devtron/pkg/cluster/repository" bean2 "github.com/devtron-labs/devtron/pkg/configDiff/bean" "github.com/devtron-labs/devtron/pkg/configDiff/helper" + "github.com/devtron-labs/devtron/pkg/configDiff/utils" + "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/bean" "go.uber.org/zap" + "net/http" + "strconv" ) type DeploymentConfigurationService interface { ConfigAutoComplete(appId int, envId int) (*bean2.ConfigDataResponse, error) + GetAllConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams) (*bean2.DeploymentAndCmCsConfigDto, error) } type DeploymentConfigurationServiceImpl struct { - logger *zap.SugaredLogger - configMapService pipeline.ConfigMapService + logger *zap.SugaredLogger + configMapService pipeline.ConfigMapService + appRepository appRepository.AppRepository + environmentRepository repository.EnvironmentRepository + chartService chartService.ChartService + deploymentTemplateService generateManifest.DeploymentTemplateService } func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, configMapService pipeline.ConfigMapService, + appRepository appRepository.AppRepository, + environmentRepository repository.EnvironmentRepository, + chartService chartService.ChartService, + deploymentTemplateService generateManifest.DeploymentTemplateService, ) (*DeploymentConfigurationServiceImpl, error) { deploymentConfigurationService := &DeploymentConfigurationServiceImpl{ - logger: logger, - configMapService: configMapService, + logger: logger, + configMapService: configMapService, + appRepository: appRepository, + environmentRepository: environmentRepository, + chartService: chartService, + deploymentTemplateService: deploymentTemplateService, } return deploymentConfigurationService, nil @@ -51,3 +75,153 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en configDataResp := bean2.NewConfigDataResponse().WithResourceConfig(combinedProperties) return configDataResp, nil } + +func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams) (*bean2.DeploymentAndCmCsConfigDto, error) { + if !configDataQueryParams.IsValidConfigType() { + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: strconv.Itoa(http.StatusBadRequest), InternalMessage: bean2.InvalidConfigTypeErr, UserMessage: bean2.InvalidConfigTypeErr} + } + var err error + var envId int + var appId int + if configDataQueryParams.IsEnvNameProvided() { + envModel, err := impl.environmentRepository.FindByName(configDataQueryParams.EnvName) + if err != nil { + impl.logger.Errorw("GetAllConfigData, error in getting environment model by envName", "envName", configDataQueryParams.EnvName, "err", err) + return nil, err + } + envId = envModel.Id + } + appModel, err := impl.appRepository.FindActiveByName(configDataQueryParams.AppName) + if err != nil { + impl.logger.Errorw("GetAllConfigData, error in getting app model by appName", "appName", configDataQueryParams.AppName, "err", err) + return nil, err + } + appId = appModel.Id + + configData := make([]*bean2.DeploymentAndCmCsConfig, 0) + switch configDataQueryParams.ConfigType { + default: // keeping default as PublishedOnly + cmcsConfigData, err := impl.getCmCsConfigResponse(configDataQueryParams, envId, appId) + if err != nil { + impl.logger.Errorw("GetAllConfigData, error in getting cm cs", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) + return nil, err + } + if configDataQueryParams.IsRequestMadeForOneResource() { + return bean2.NewDeploymentAndCmCsConfigDto().WithConfigData(configData).WithAppAndEnvIdId(appId, envId), nil + } + configData = append(configData, cmcsConfigData...) + + deploymentTemplateJsonData, err := impl.getDeploymentConfig(ctx, appId, envId) + if err != nil { + impl.logger.Errorw("GetAllConfigData, error in getting deployment config", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) + return nil, err + } + deploymentConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigState(bean2.PublishedConfigState).WithConfigData(deploymentTemplateJsonData).WithResourceType(bean.DeploymentTemplate) + configData = append(configData, deploymentConfig) + } + + return bean2.NewDeploymentAndCmCsConfigDto().WithConfigData(configData).WithAppAndEnvIdId(appId, envId), nil +} + +func (impl *DeploymentConfigurationServiceImpl) getBaseDeploymentTemplate(appId int) (json.RawMessage, error) { + deploymentTemplateData, err := impl.chartService.FindLatestChartForAppByAppId(appId) + if err != nil { + impl.logger.Errorw("error in getting base deployment template for appId", "appId", appId, "err", err) + return nil, err + } + return deploymentTemplateData.DefaultAppOverride, nil +} + +func (impl *DeploymentConfigurationServiceImpl) getDeploymentTemplateForEnvLevel(ctx context.Context, appId, envId int) (json.RawMessage, error) { + deploymentTemplateRequest := generateManifest.DeploymentTemplateRequest{ + AppId: appId, + EnvId: envId, + RequestDataMode: generateManifest.Values, + Type: repository2.PublishedOnEnvironments, + } + deploymentTemplateResponse, err := impl.deploymentTemplateService.GetDeploymentTemplate(ctx, deploymentTemplateRequest) + if err != nil { + impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in getting deployment template for ", "deploymentTemplateRequest", deploymentTemplateRequest, "err", err) + return nil, err + } + deploymentJson := json.RawMessage{} + err = deploymentJson.UnmarshalJSON([]byte(deploymentTemplateResponse.Data)) + if err != nil { + impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in unmarshalling string deploymentTemplateResponse data into json Raw message", "data", deploymentTemplateResponse.Data, "err", err) + return nil, err + } + return deploymentJson, nil +} + +func (impl *DeploymentConfigurationServiceImpl) getDeploymentConfig(ctx context.Context, appId, envId int) (json.RawMessage, error) { + if envId > 0 { + return impl.getDeploymentTemplateForEnvLevel(ctx, appId, envId) + } + return impl.getBaseDeploymentTemplate(appId) +} + +func (impl *DeploymentConfigurationServiceImpl) getSecretConfigResponse(resourceName string, envId, appId int) (*bean.ConfigDataRequest, error) { + if len(resourceName) > 0 { + if envId > 0 { + return impl.configMapService.CSEnvironmentFetchForEdit(resourceName, 0, appId, envId) + } + return impl.configMapService.CSGlobalFetchForEditUsingAppId(resourceName, appId) + } + + if envId > 0 { + return impl.configMapService.CSEnvironmentFetch(appId, envId) + } + return impl.configMapService.CSGlobalFetch(appId) +} + +func (impl *DeploymentConfigurationServiceImpl) getConfigMapResponse(resourceName string, envId, appId int) (*bean.ConfigDataRequest, error) { + if len(resourceName) > 0 { + if envId > 0 { + return impl.configMapService.CMEnvironmentFetchForEdit(resourceName, 0, appId, envId) + } + return impl.configMapService.CMGlobalFetchForEditUsingAppId(resourceName, appId) + } + + if envId > 0 { + return impl.configMapService.CMEnvironmentFetch(appId, envId) + } + return impl.configMapService.CMGlobalFetch(appId) +} + +func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigResponse(configDataQueryParams *bean2.ConfigDataQueryParams, envId, appId int) ([]*bean2.DeploymentAndCmCsConfig, error) { + + configData := make([]*bean2.DeploymentAndCmCsConfig, 0) + var cmcsRespJson json.RawMessage + var resourceType bean.ResourceType + cmcsConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigState(bean2.PublishedConfigState) + + if configDataQueryParams.IsResourceTypeSecret() { + secretData, err := impl.getSecretConfigResponse(configDataQueryParams.ResourceName, envId, appId) + if err != nil { + impl.logger.Errorw("getCmCsConfigResponse, error in getting secret config response for appName and envName", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) + return nil, err + } + resourceType = bean.CS + cmcsRespJson, err = utils.ConvertToJsonRawMessage(secretData) + if err != nil { + impl.logger.Errorw("getCmCsConfigResponse, error in getting secret json raw message", "err", err) + return nil, err + } + } else if configDataQueryParams.IsResourceTypeConfigMap() { + cmData, err := impl.getConfigMapResponse(configDataQueryParams.ResourceName, envId, appId) + if err != nil { + impl.logger.Errorw("getCmCsConfigResponse, error in getting config map for appName and envName", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) + return nil, err + } + resourceType = bean.CM + cmcsRespJson, err = utils.ConvertToJsonRawMessage(cmData) + if err != nil { + impl.logger.Errorw("getCmCsConfigResponse, error in getting cm json raw message", "cmData", cmData, "err", err) + return nil, err + } + } + cmcsConfig.WithConfigData(cmcsRespJson).WithResourceType(resourceType) + configData = append(configData, cmcsConfig) + + return configData, nil +} diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 1a6eedd62bc..483651e9efd 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -1,16 +1,22 @@ package bean import ( + "encoding/json" "fmt" + v1 "github.com/devtron-labs/devtron/pkg/apis/devtron/v1" "github.com/devtron-labs/devtron/pkg/pipeline/bean" ) -type ConfigState int +type ConfigState string const ( - PublishedConfigState ConfigState = 3 + PublishedConfigState ConfigState = "PublishedOnly" ) +func (r ConfigState) ToString() string { + return string(r) +} + type ConfigProperty struct { Id int `json:"id"` Name string `json:"name"` @@ -67,15 +73,96 @@ func (r *ConfigProperty) GetIdentifier() ConfigPropertyIdentifier { } type DeploymentAndCmCsConfig struct { - Id int `json:"id ,omitempty"` - Name string `json:"name"` - CmCsType string `json:"cmcsType ,omitempty"` //example:- environment or volume, in case of cm cs - Data string `json:"data"` - ConfigState ConfigState `json:"configState"` + Id int `json:"id ,omitempty"` + Name string `json:"name"` + ResourceType bean.ResourceType `json:"resourceType"` + Data json.RawMessage `json:"data"` + ConfigState ConfigState `json:"configState"` +} + +func NewDeploymentAndCmCsConfig() *DeploymentAndCmCsConfig { + return &DeploymentAndCmCsConfig{} +} + +func (r *DeploymentAndCmCsConfig) WithIdAndName(id int, name string) *DeploymentAndCmCsConfig { + r.Id = id + r.Name = name + return r +} + +func (r *DeploymentAndCmCsConfig) WithConfigState(configState ConfigState) *DeploymentAndCmCsConfig { + r.ConfigState = configState + return r } -type DeploymentAndCmCsResponseDto struct { +func (r *DeploymentAndCmCsConfig) WithResourceType(resourceType bean.ResourceType) *DeploymentAndCmCsConfig { + r.ResourceType = resourceType + return r +} + +func (r *DeploymentAndCmCsConfig) WithConfigData(data json.RawMessage) *DeploymentAndCmCsConfig { + r.Data = data + return r +} + +type DeploymentAndCmCsConfigDto struct { AppId int `json:"appId"` EnvironmentId int `json:"environmentId"` ConfigData []*DeploymentAndCmCsConfig `json:"configData"` } + +func NewDeploymentAndCmCsConfigDto() *DeploymentAndCmCsConfigDto { + return &DeploymentAndCmCsConfigDto{ConfigData: make([]*DeploymentAndCmCsConfig, 0)} +} + +func (r *DeploymentAndCmCsConfigDto) WithConfigData(configData []*DeploymentAndCmCsConfig) *DeploymentAndCmCsConfigDto { + r.ConfigData = configData + return r +} + +func (r *DeploymentAndCmCsConfigDto) WithAppAndEnvIdId(appId, envId int) *DeploymentAndCmCsConfigDto { + r.AppId = appId + r.EnvironmentId = envId + return r +} + +type ConfigDataQueryParams struct { + AppName string + EnvName string + ConfigType string + IdentifierId int + ResourceName string + ResourceType string +} + +func (r *ConfigDataQueryParams) IsResourceTypeSecret() bool { + return r.ResourceType == v1.Secret +} + +func (r *ConfigDataQueryParams) IsResourceTypeConfigMap() bool { + return r.ResourceType == v1.ConfigMap +} + +func (r *ConfigDataQueryParams) IsEnvNameProvided() bool { + return len(r.EnvName) > 0 +} +func (r *ConfigDataQueryParams) IsValidConfigType() bool { + return r.ConfigType == PublishedConfigState.ToString() +} + +func (r *ConfigDataQueryParams) IsRequestMadeForOneResource() bool { + return len(r.ResourceName) > 0 && len(r.ResourceType) > 0 +} + +func (r *ConfigDataQueryParams) WithConfigDataQueryParams(appName, envName, configType, resourceName, resourceType string, identifierId int) { + r.AppName = appName + r.EnvName = envName + r.ConfigType = configType + r.ResourceName = resourceName + r.IdentifierId = identifierId + r.ResourceType = resourceType +} + +const ( + InvalidConfigTypeErr = "invalid config type provided, please send a valid config type" +) diff --git a/pkg/configDiff/utils/utils.go b/pkg/configDiff/utils/utils.go new file mode 100644 index 00000000000..8185993775f --- /dev/null +++ b/pkg/configDiff/utils/utils.go @@ -0,0 +1,16 @@ +package utils + +import "encoding/json" + +func ConvertToJsonRawMessage(request interface{}) (json.RawMessage, error) { + var r json.RawMessage + configMapByte, err := json.Marshal(request) + if err != nil { + return nil, err + } + err = r.UnmarshalJSON(configMapByte) + if err != nil { + return nil, err + } + return r, nil +} diff --git a/pkg/pipeline/ConfigMapService.go b/pkg/pipeline/ConfigMapService.go index 975dfd96cc5..bb2d7334d7d 100644 --- a/pkg/pipeline/ConfigMapService.go +++ b/pkg/pipeline/ConfigMapService.go @@ -56,6 +56,7 @@ type ConfigMapService interface { CMEnvironmentFetch(appId int, envId int) (*bean.ConfigDataRequest, error) CMGlobalFetchForEdit(name string, id int) (*bean.ConfigDataRequest, error) CMEnvironmentFetchForEdit(name string, id int, appId int, envId int) (*bean.ConfigDataRequest, error) + CMGlobalFetchForEditUsingAppId(name string, appId int) (*bean.ConfigDataRequest, error) CSGlobalAddUpdate(configMapRequest *bean.ConfigDataRequest) (*bean.ConfigDataRequest, error) CSGlobalFetch(appId int) (*bean.ConfigDataRequest, error) @@ -72,6 +73,7 @@ type ConfigMapService interface { CSGlobalDeleteByAppId(name string, appId int, userId int32) (bool, error) CSEnvironmentDeleteByAppIdAndEnvId(name string, appId int, envId int, userId int32) (bool, error) + CSGlobalFetchForEditUsingAppId(name string, appId int) (*bean.ConfigDataRequest, error) CSGlobalFetchForEdit(name string, id int) (*bean.ConfigDataRequest, error) CSEnvironmentFetchForEdit(name string, id int, appId int, envId int) (*bean.ConfigDataRequest, error) ConfigSecretGlobalBulkPatch(bulkPatchRequest *bean.BulkPatchRequest) (*bean.BulkPatchRequest, error) @@ -2010,3 +2012,33 @@ func (impl ConfigMapServiceImpl) FetchCmCsNamesAppAndEnvLevel(appId int, envId i } return cMCSNamesAppLevel, cMCSNamesEnvLevel, nil } + +func (impl ConfigMapServiceImpl) CMGlobalFetchForEditUsingAppId(name string, appId int) (*bean.ConfigDataRequest, error) { + configDataRequest, err := impl.CMGlobalFetch(appId) + if err != nil { + return nil, err + } + var configs []*bean.ConfigData + for _, configData := range configDataRequest.ConfigData { + if configData.Name == name { + configs = append(configs, configData) + } + } + configDataRequest.ConfigData = configs + return configDataRequest, nil +} + +func (impl ConfigMapServiceImpl) CSGlobalFetchForEditUsingAppId(name string, appId int) (*bean.ConfigDataRequest, error) { + configDataRequest, err := impl.CSGlobalFetch(appId) + if err != nil { + return nil, err + } + var configs []*bean.ConfigData + for _, configData := range configDataRequest.ConfigData { + if configData.Name == name { + configs = append(configs, configData) + } + } + configDataRequest.ConfigData = configs + return configDataRequest, nil +} diff --git a/wire_gen.go b/wire_gen.go index ccd38fe6f91..83bfc68de23 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -928,7 +928,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl) + deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl) if err != nil { return nil, err } From 2c5c7c62a46d28ea237f29d06ef5d05ffd0a457b Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 17 Jul 2024 15:53:41 +0530 Subject: [PATCH 29/45] using a single key instead of global and overridden key in config/autocomplete api --- .../DeploymentConfigurationService.go | 7 ++--- pkg/configDiff/bean/bean.go | 26 +++++++------------ pkg/configDiff/helper/helper.go | 2 +- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 2df817b80b1..fc9247f5ea4 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -36,13 +36,14 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap := helper.GetCmCsAppAndEnvLevelMap(cMCSNamesAppLevel, cMCSNamesEnvLevel) for key, configProperty := range cmcsKeyPropertyAppLevelMap { if _, ok := cmcsKeyPropertyEnvLevelMap[key]; !ok { - configProperty.Global = true + configProperty.ConfigStage = bean2.Inheriting } } for key, configProperty := range cmcsKeyPropertyEnvLevelMap { if _, ok := cmcsKeyPropertyAppLevelMap[key]; ok { - configProperty.Global = true - configProperty.Overridden = true + configProperty.ConfigStage = bean2.Overridden + } else { + configProperty.ConfigStage = bean2.Env } } combinedProperties := helper.GetCombinedPropertiesMap(cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap) diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index c7f1d538ea9..9422de8685e 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -11,30 +11,24 @@ const ( PublishedConfigState ConfigState = 3 ) +type ConfigStage string + +const ( + Env ConfigStage = "Env" + Inheriting ConfigStage = "Inheriting" + Overridden ConfigStage = "Overridden" +) + type ConfigProperty struct { Id int `json:"id"` Name string `json:"name"` ConfigState ConfigState `json:"configState"` Type bean.ResourceType `json:"type"` - Overridden bool `json:"overridden"` - Global bool `json:"global"` -} - -func NewConfigProperty() *ConfigProperty { - return &ConfigProperty{} + ConfigStage ConfigStage `json:"configStage"` } func (r *ConfigProperty) IsConfigPropertyGlobal() bool { - return r.Global -} - -func (r *ConfigProperty) SetConfigProperty(Name string, ConfigState ConfigState, Type bean.ResourceType, Overridden bool, Global bool) *ConfigProperty { - r.Name = Name - r.ConfigState = ConfigState - r.Type = Type - r.Overridden = Overridden - r.Global = Global - return r + return r.ConfigStage == Inheriting } type ConfigDataResponse struct { diff --git a/pkg/configDiff/helper/helper.go b/pkg/configDiff/helper/helper.go index 3182cc16d9a..9d47d9f5489 100644 --- a/pkg/configDiff/helper/helper.go +++ b/pkg/configDiff/helper/helper.go @@ -32,7 +32,7 @@ func GetCombinedPropertiesMap(cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLeve combinedPropertiesMap := make(map[string]*bean2.ConfigProperty, len(cmcsKeyPropertyAppLevelMap)+len(cmcsKeyPropertyEnvLevelMap)) for key, property := range cmcsKeyPropertyAppLevelMap { if property.IsConfigPropertyGlobal() { - // only append global =true in combined for app level cmcs, all overridden cmcs would be handled in cmcsKeyPropertyEnvLevelMap + // only append inheriting in combined for app level cmcs, all overridden cmcs would be handled in cmcsKeyPropertyEnvLevelMap combinedPropertiesMap[key] = property } } From 8696b08f8841f9594df30d965395d43e11c379d3 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 17 Jul 2024 16:20:36 +0530 Subject: [PATCH 30/45] ConfigState made string instead of int --- pkg/configDiff/bean/bean.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 9422de8685e..0162f6ce03f 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -5,10 +5,10 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/bean" ) -type ConfigState int +type ConfigState string const ( - PublishedConfigState ConfigState = 3 + PublishedConfigState ConfigState = "Published" ) type ConfigStage string From f9d9c6527eb24e142a06a78ef0a641f10f56f2b3 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 24 Jul 2024 00:20:06 +0530 Subject: [PATCH 31/45] not sending inheriting in case base config --- pkg/configDiff/DeploymentConfigurationService.go | 5 ++++- pkg/configDiff/helper/helper.go | 5 +---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index fc9247f5ea4..13900dd4fc5 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -36,7 +36,10 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap := helper.GetCmCsAppAndEnvLevelMap(cMCSNamesAppLevel, cMCSNamesEnvLevel) for key, configProperty := range cmcsKeyPropertyAppLevelMap { if _, ok := cmcsKeyPropertyEnvLevelMap[key]; !ok { - configProperty.ConfigStage = bean2.Inheriting + if envId > 0 { + configProperty.ConfigStage = bean2.Inheriting + } + } } for key, configProperty := range cmcsKeyPropertyEnvLevelMap { diff --git a/pkg/configDiff/helper/helper.go b/pkg/configDiff/helper/helper.go index 9d47d9f5489..1ce74297eaa 100644 --- a/pkg/configDiff/helper/helper.go +++ b/pkg/configDiff/helper/helper.go @@ -31,10 +31,7 @@ func GetConfigProperty(id int, name string, configType bean.ResourceType, State func GetCombinedPropertiesMap(cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap map[string]*bean2.ConfigProperty) []*bean2.ConfigProperty { combinedPropertiesMap := make(map[string]*bean2.ConfigProperty, len(cmcsKeyPropertyAppLevelMap)+len(cmcsKeyPropertyEnvLevelMap)) for key, property := range cmcsKeyPropertyAppLevelMap { - if property.IsConfigPropertyGlobal() { - // only append inheriting in combined for app level cmcs, all overridden cmcs would be handled in cmcsKeyPropertyEnvLevelMap - combinedPropertiesMap[key] = property - } + combinedPropertiesMap[key] = property } for key, property := range cmcsKeyPropertyEnvLevelMap { combinedPropertiesMap[key] = property From 5628d6cdabd51695dd5223f05873f99bbf4dc54f Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 30 Jul 2024 18:15:23 +0530 Subject: [PATCH 32/45] code review comment incorporation --- .../DeploymentConfigurationRestHandler.go | 2 +- cmd/external-app/wire_gen.go | 2 +- .../chartConfig/ConfigMapRepository.go | 1 + .../DeploymentConfigurationService.go | 5 ++-- pkg/configDiff/adaptor/adaptor.go | 29 +++++++++++++++++++ pkg/configDiff/helper/helper.go | 24 --------------- wire_gen.go | 2 +- 7 files changed, 36 insertions(+), 29 deletions(-) create mode 100644 pkg/configDiff/adaptor/adaptor.go diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 567ff9895ba..216a00d8bbd 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -63,7 +63,7 @@ func (handler *DeploymentConfigurationRestHandlerImpl) ConfigAutoComplete(w http res, err := handler.deploymentConfigurationService.ConfigAutoComplete(appId, envId) if err != nil { - handler.logger.Errorw("service err, ConfigAutoComplete ", "err", err) + handler.logger.Errorw("service err, ConfigAutoComplete ", "appId", appId, "envId", envId, "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 917252738dd..7cada24e76b 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/internal/sql/repository/chartConfig/ConfigMapRepository.go b/internal/sql/repository/chartConfig/ConfigMapRepository.go index 67f2edec141..764378765c3 100644 --- a/internal/sql/repository/chartConfig/ConfigMapRepository.go +++ b/internal/sql/repository/chartConfig/ConfigMapRepository.go @@ -76,6 +76,7 @@ func (impl ConfigMapRepositoryImpl) GetConfigNamesForAppAndEnvLevel(appId int, e if envId == -1 { tableName = ConfigMapAppLevel } + //below query iterates over the cm, cs stored as json element, and fetches cmName and csName, id for a particular appId or envId if provided query := impl.dbConnection. Model(). Table(tableName). diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 13900dd4fc5..78a25a8ee25 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -1,6 +1,7 @@ package configDiff import ( + "github.com/devtron-labs/devtron/pkg/configDiff/adaptor" bean2 "github.com/devtron-labs/devtron/pkg/configDiff/bean" "github.com/devtron-labs/devtron/pkg/configDiff/helper" "github.com/devtron-labs/devtron/pkg/pipeline" @@ -33,7 +34,7 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en impl.logger.Errorw("error in fetching CM and CS names at app or env level", "appId", appId, "envId", envId, "err", err) return nil, err } - cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap := helper.GetCmCsAppAndEnvLevelMap(cMCSNamesAppLevel, cMCSNamesEnvLevel) + cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap := adaptor.GetCmCsAppAndEnvLevelMap(cMCSNamesAppLevel, cMCSNamesEnvLevel) for key, configProperty := range cmcsKeyPropertyAppLevelMap { if _, ok := cmcsKeyPropertyEnvLevelMap[key]; !ok { if envId > 0 { @@ -50,7 +51,7 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en } } combinedProperties := helper.GetCombinedPropertiesMap(cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap) - combinedProperties = append(combinedProperties, helper.GetConfigProperty(0, "", bean.DeploymentTemplate, bean2.PublishedConfigState)) + combinedProperties = append(combinedProperties, adaptor.GetConfigProperty(0, "", bean.DeploymentTemplate, bean2.PublishedConfigState)) configDataResp := bean2.NewConfigDataResponse().WithResourceConfig(combinedProperties) return configDataResp, nil diff --git a/pkg/configDiff/adaptor/adaptor.go b/pkg/configDiff/adaptor/adaptor.go new file mode 100644 index 00000000000..4ab81eb2d11 --- /dev/null +++ b/pkg/configDiff/adaptor/adaptor.go @@ -0,0 +1,29 @@ +package adaptor + +import ( + bean2 "github.com/devtron-labs/devtron/pkg/configDiff/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" +) + +func GetConfigProperty(id int, name string, configType bean.ResourceType, State bean2.ConfigState) *bean2.ConfigProperty { + return &bean2.ConfigProperty{ + Id: id, + Name: name, + Type: configType, + ConfigState: State, + } +} + +func GetCmCsAppAndEnvLevelMap(cMCSNamesAppLevel, cMCSNamesEnvLevel []bean.ConfigNameAndType) (map[string]*bean2.ConfigProperty, map[string]*bean2.ConfigProperty) { + cMCSNamesAppLevelMap, cMCSNamesEnvLevelMap := make(map[string]*bean2.ConfigProperty, len(cMCSNamesAppLevel)), make(map[string]*bean2.ConfigProperty, len(cMCSNamesEnvLevel)) + + for _, cmcs := range cMCSNamesAppLevel { + property := GetConfigProperty(cmcs.Id, cmcs.Name, cmcs.Type, bean2.PublishedConfigState) + cMCSNamesAppLevelMap[property.GetKey()] = property + } + for _, cmcs := range cMCSNamesEnvLevel { + property := GetConfigProperty(cmcs.Id, cmcs.Name, cmcs.Type, bean2.PublishedConfigState) + cMCSNamesEnvLevelMap[property.GetKey()] = property + } + return cMCSNamesAppLevelMap, cMCSNamesEnvLevelMap +} diff --git a/pkg/configDiff/helper/helper.go b/pkg/configDiff/helper/helper.go index 1ce74297eaa..70082a7bea6 100644 --- a/pkg/configDiff/helper/helper.go +++ b/pkg/configDiff/helper/helper.go @@ -2,32 +2,8 @@ package helper import ( bean2 "github.com/devtron-labs/devtron/pkg/configDiff/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/bean" ) -func GetCmCsAppAndEnvLevelMap(cMCSNamesAppLevel, cMCSNamesEnvLevel []bean.ConfigNameAndType) (map[string]*bean2.ConfigProperty, map[string]*bean2.ConfigProperty) { - cMCSNamesAppLevelMap, cMCSNamesEnvLevelMap := make(map[string]*bean2.ConfigProperty, len(cMCSNamesAppLevel)), make(map[string]*bean2.ConfigProperty, len(cMCSNamesEnvLevel)) - - for _, cmcs := range cMCSNamesAppLevel { - property := GetConfigProperty(cmcs.Id, cmcs.Name, cmcs.Type, bean2.PublishedConfigState) - cMCSNamesAppLevelMap[property.GetKey()] = property - } - for _, cmcs := range cMCSNamesEnvLevel { - property := GetConfigProperty(cmcs.Id, cmcs.Name, cmcs.Type, bean2.PublishedConfigState) - cMCSNamesEnvLevelMap[property.GetKey()] = property - } - return cMCSNamesAppLevelMap, cMCSNamesEnvLevelMap -} - -func GetConfigProperty(id int, name string, configType bean.ResourceType, State bean2.ConfigState) *bean2.ConfigProperty { - return &bean2.ConfigProperty{ - Id: id, - Name: name, - Type: configType, - ConfigState: State, - } -} - func GetCombinedPropertiesMap(cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap map[string]*bean2.ConfigProperty) []*bean2.ConfigProperty { combinedPropertiesMap := make(map[string]*bean2.ConfigProperty, len(cmcsKeyPropertyAppLevelMap)+len(cmcsKeyPropertyEnvLevelMap)) for key, property := range cmcsKeyPropertyAppLevelMap { diff --git a/wire_gen.go b/wire_gen.go index 2b4256c046c..d033ed142a4 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject From 7f29f200f73b5a97f49a6a09db31b7c16b4ce15b Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 31 Jul 2024 10:14:00 +0530 Subject: [PATCH 33/45] ent sync --- .../DeploymentConfigurationRestHandler.go | 32 +++- cmd/external-app/wire_gen.go | 2 +- env_gen.md | 1 + .../DeploymentConfigurationService.go | 175 ++++++++++++------ pkg/configDiff/bean/bean.go | 70 +++---- 5 files changed, 171 insertions(+), 109 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index c59f45212a1..8d1cc2ce728 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -108,18 +108,42 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp func getConfigDataQueryParams(r *http.Request) (*bean.ConfigDataQueryParams, error) { v := r.URL.Query() + + var identifierId int + var pipelineId int + var resourceId int + var err error + appName := v.Get("appName") envName := v.Get("envName") configType := v.Get("configType") identifierIdStr := v.Get("identifierId") - identifierId, err := strconv.Atoi(identifierIdStr) - if err != nil { - return nil, err + if len(identifierIdStr) > 0 { + identifierId, err = strconv.Atoi(identifierIdStr) + if err != nil { + return nil, err + } + } + + pipelineIdStr := v.Get("pipelineId") + + if len(pipelineIdStr) > 0 { + pipelineId, err = strconv.Atoi(pipelineIdStr) + if err != nil { + return nil, err + } } resourceName := v.Get("resourceName") resourceType := v.Get("resourceType") + resourceIdStr := v.Get("resourceId") + if len(resourceIdStr) > 0 { + resourceId, err = strconv.Atoi(resourceIdStr) + if err != nil { + return nil, err + } + } return &bean.ConfigDataQueryParams{ AppName: appName, EnvName: envName, @@ -127,5 +151,7 @@ func getConfigDataQueryParams(r *http.Request) (*bean.ConfigDataQueryParams, err IdentifierId: identifierId, ResourceName: resourceName, ResourceType: resourceType, + PipelineId: pipelineId, + ResourceId: resourceId, }, nil } diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 917252738dd..7cada24e76b 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/env_gen.md b/env_gen.md index defe6ed8d26..8b12c396a19 100644 --- a/env_gen.md +++ b/env_gen.md @@ -9,6 +9,7 @@ | APP | orchestrator | | | APP_SYNC_IMAGE | quay.io/devtron/chart-sync:1227622d-132-3775 | | | APP_SYNC_JOB_RESOURCES_OBJ | | | + | APP_SYNC_SERVICE_ACCOUNT | chart-sync | | | ARGO_APP_MANUAL_SYNC_TIME | 3 | | | ARGO_AUTO_SYNC_ENABLED | true | | | ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT | 3 | | diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 0ad0c1cdfcb..6be810832c3 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -60,13 +60,17 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap := helper.GetCmCsAppAndEnvLevelMap(cMCSNamesAppLevel, cMCSNamesEnvLevel) for key, configProperty := range cmcsKeyPropertyAppLevelMap { if _, ok := cmcsKeyPropertyEnvLevelMap[key]; !ok { - configProperty.Global = true + if envId > 0 { + configProperty.ConfigStage = bean2.Inheriting + } + } } for key, configProperty := range cmcsKeyPropertyEnvLevelMap { if _, ok := cmcsKeyPropertyAppLevelMap[key]; ok { - configProperty.Global = true - configProperty.Overridden = true + configProperty.ConfigStage = bean2.Overridden + } else { + configProperty.ConfigStage = bean2.Env } } combinedProperties := helper.GetCombinedPropertiesMap(cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap) @@ -98,29 +102,118 @@ func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Con } appId = appModel.Id - configData := make([]*bean2.DeploymentAndCmCsConfig, 0) + configDataDto := &bean2.DeploymentAndCmCsConfigDto{} switch configDataQueryParams.ConfigType { default: // keeping default as PublishedOnly - cmcsConfigData, err := impl.getCmCsConfigResponse(configDataQueryParams, envId, appId) + configDataDto, err = impl.getPublishedConfigData(ctx, configDataQueryParams, appId, envId) if err != nil { - impl.logger.Errorw("GetAllConfigData, error in getting cm cs", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) + impl.logger.Errorw("GetAllConfigData, error in config data for PublishedOnly", "configDataQueryParams", configDataQueryParams, "err", err) return nil, err } - if configDataQueryParams.IsRequestMadeForOneResource() { - return bean2.NewDeploymentAndCmCsConfigDto().WithConfigData(configData).WithAppAndEnvIdId(appId, envId), nil - } - configData = append(configData, cmcsConfigData...) + } + return configDataDto, nil +} - deploymentTemplateJsonData, err := impl.getDeploymentConfig(ctx, appId, envId) - if err != nil { - impl.logger.Errorw("GetAllConfigData, error in getting deployment config", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) - return nil, err - } - deploymentConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigState(bean2.PublishedConfigState).WithConfigData(deploymentTemplateJsonData).WithResourceType(bean.DeploymentTemplate) - configData = append(configData, deploymentConfig) +func (impl *DeploymentConfigurationServiceImpl) getCmCsEditDataForPublishedOnly(configDataQueryParams *bean2.ConfigDataQueryParams, envId, appId int) (*bean2.DeploymentAndCmCsConfigDto, error) { + configDataDto := &bean2.DeploymentAndCmCsConfigDto{} + + var resourceType bean.ResourceType + var fetchConfigFunc func(string, int, int, int) (*bean.ConfigDataRequest, error) + + if configDataQueryParams.IsResourceTypeSecret() { + //handles for single resource when resource type is secret and for a given resource name + resourceType = bean.CS + fetchConfigFunc = impl.getSecretConfigResponse + } else if configDataQueryParams.IsResourceTypeConfigMap() { + //handles for single resource when resource type is configMap and for a given resource name + resourceType = bean.CM + fetchConfigFunc = impl.getConfigMapResponse + } + cmcsConfigData, err := fetchConfigFunc(configDataQueryParams.ResourceName, configDataQueryParams.ResourceId, envId, appId) + if err != nil { + impl.logger.Errorw("getCmCsEditDataForPublishedOnly, error in getting config response", "resourceName", configDataQueryParams.ResourceName, "envName", configDataQueryParams.EnvName, "err", err) + return nil, err } - return bean2.NewDeploymentAndCmCsConfigDto().WithConfigData(configData).WithAppAndEnvIdId(appId, envId), nil + respJson, err := utils.ConvertToJsonRawMessage(cmcsConfigData) + if err != nil { + impl.logger.Errorw("getCmCsEditDataForPublishedOnly, error in converting to json raw message", "configDataQueryParams", configDataQueryParams, "err", err) + return nil, err + } + + cmCsConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigData(respJson).WithResourceType(resourceType) + if resourceType == bean.CS { + configDataDto.WithSecretData(cmCsConfig) + } else if resourceType == bean.CM { + configDataDto.WithConfigMapData(cmCsConfig) + } + return configDataDto, nil +} + +func (impl *DeploymentConfigurationServiceImpl) getCmCsPublishedConfigResponse(envId, appId int) (*bean2.DeploymentAndCmCsConfigDto, error) { + + configDataDto := &bean2.DeploymentAndCmCsConfigDto{} + secretData, err := impl.getSecretConfigResponse("", 0, envId, appId) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in getting secret config response by appId and envId", "appId", appId, "envId", envId, "err", err) + return nil, err + } + + //iterate on secret configData and then and set draft data from draftResourcesMap if same resourceName found do the same for configMap below + cmData, err := impl.getConfigMapResponse("", 0, envId, appId) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in getting config map by appId and envId", "appId", appId, "envId", envId, "err", err) + return nil, err + } + + secretRespJson, err := utils.ConvertToJsonRawMessage(secretData) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in converting secret data to json raw message", "appId", appId, "envId", envId, "err", err) + return nil, err + } + + cmRespJson, err := utils.ConvertToJsonRawMessage(cmData) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in converting config map data to json raw message", "appId", appId, "envId", envId, "err", err) + return nil, err + } + + cmConfigData := bean2.NewDeploymentAndCmCsConfig().WithConfigData(cmRespJson).WithResourceType(bean.CM) + secretConfigData := bean2.NewDeploymentAndCmCsConfig().WithConfigData(secretRespJson).WithResourceType(bean.CS) + + configDataDto.WithConfigMapData(cmConfigData).WithSecretData(secretConfigData) + return configDataDto, nil + +} + +func (impl *DeploymentConfigurationServiceImpl) getPublishedDeploymentConfig(ctx context.Context, appId, envId int) (json.RawMessage, error) { + if envId > 0 { + return impl.getDeploymentTemplateForEnvLevel(ctx, appId, envId) + } + return impl.getBaseDeploymentTemplate(appId) +} + +func (impl *DeploymentConfigurationServiceImpl) getPublishedConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, + appId, envId int) (*bean2.DeploymentAndCmCsConfigDto, error) { + + if configDataQueryParams.IsRequestMadeForOneResource() { + return impl.getCmCsEditDataForPublishedOnly(configDataQueryParams, envId, appId) + } + //ConfigMapsData and SecretsData are populated here + configData, err := impl.getCmCsPublishedConfigResponse(envId, appId) + if err != nil { + impl.logger.Errorw("getPublishedConfigData, error in getting cm cs for PublishedOnly state", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) + return nil, err + } + deploymentTemplateJsonData, err := impl.getPublishedDeploymentConfig(ctx, appId, envId) + if err != nil { + impl.logger.Errorw("getPublishedConfigData, error in getting publishedOnly deployment config ", "configDataQueryParams", configDataQueryParams, "err", err) + return nil, err + } + deploymentConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigData(deploymentTemplateJsonData).WithResourceType(bean.DeploymentTemplate) + + configData.WithDeploymentTemplateData(deploymentConfig) + return configData, nil } func (impl *DeploymentConfigurationServiceImpl) getBaseDeploymentTemplate(appId int) (json.RawMessage, error) { @@ -160,10 +253,10 @@ func (impl *DeploymentConfigurationServiceImpl) getDeploymentConfig(ctx context. return impl.getBaseDeploymentTemplate(appId) } -func (impl *DeploymentConfigurationServiceImpl) getSecretConfigResponse(resourceName string, envId, appId int) (*bean.ConfigDataRequest, error) { +func (impl *DeploymentConfigurationServiceImpl) getSecretConfigResponse(resourceName string, resourceId, envId, appId int) (*bean.ConfigDataRequest, error) { if len(resourceName) > 0 { if envId > 0 { - return impl.configMapService.CSEnvironmentFetchForEdit(resourceName, 0, appId, envId) + return impl.configMapService.CSEnvironmentFetchForEdit(resourceName, resourceId, appId, envId) } return impl.configMapService.CSGlobalFetchForEditUsingAppId(resourceName, appId) } @@ -174,10 +267,10 @@ func (impl *DeploymentConfigurationServiceImpl) getSecretConfigResponse(resource return impl.configMapService.CSGlobalFetch(appId) } -func (impl *DeploymentConfigurationServiceImpl) getConfigMapResponse(resourceName string, envId, appId int) (*bean.ConfigDataRequest, error) { +func (impl *DeploymentConfigurationServiceImpl) getConfigMapResponse(resourceName string, resourceId, envId, appId int) (*bean.ConfigDataRequest, error) { if len(resourceName) > 0 { if envId > 0 { - return impl.configMapService.CMEnvironmentFetchForEdit(resourceName, 0, appId, envId) + return impl.configMapService.CMEnvironmentFetchForEdit(resourceName, resourceId, appId, envId) } return impl.configMapService.CMGlobalFetchForEditUsingAppId(resourceName, appId) } @@ -187,41 +280,3 @@ func (impl *DeploymentConfigurationServiceImpl) getConfigMapResponse(resourceNam } return impl.configMapService.CMGlobalFetch(appId) } - -func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigResponse(configDataQueryParams *bean2.ConfigDataQueryParams, envId, appId int) ([]*bean2.DeploymentAndCmCsConfig, error) { - - configData := make([]*bean2.DeploymentAndCmCsConfig, 0) - var cmcsRespJson json.RawMessage - var resourceType bean.ResourceType - cmcsConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigState(bean2.PublishedConfigState) - - if configDataQueryParams.IsResourceTypeSecret() { - secretData, err := impl.getSecretConfigResponse(configDataQueryParams.ResourceName, envId, appId) - if err != nil { - impl.logger.Errorw("getCmCsConfigResponse, error in getting secret config response for appName and envName", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) - return nil, err - } - resourceType = bean.CS - cmcsRespJson, err = utils.ConvertToJsonRawMessage(secretData) - if err != nil { - impl.logger.Errorw("getCmCsConfigResponse, error in getting secret json raw message", "err", err) - return nil, err - } - } else if configDataQueryParams.IsResourceTypeConfigMap() { - cmData, err := impl.getConfigMapResponse(configDataQueryParams.ResourceName, envId, appId) - if err != nil { - impl.logger.Errorw("getCmCsConfigResponse, error in getting config map for appName and envName", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) - return nil, err - } - resourceType = bean.CM - cmcsRespJson, err = utils.ConvertToJsonRawMessage(cmData) - if err != nil { - impl.logger.Errorw("getCmCsConfigResponse, error in getting cm json raw message", "cmData", cmData, "err", err) - return nil, err - } - } - cmcsConfig.WithConfigData(cmcsRespJson).WithResourceType(resourceType) - configData = append(configData, cmcsConfig) - - return configData, nil -} diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 483651e9efd..460d12b267f 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -17,13 +17,20 @@ func (r ConfigState) ToString() string { return string(r) } +type ConfigStage string + +const ( + Env ConfigStage = "Env" + Inheriting ConfigStage = "Inheriting" + Overridden ConfigStage = "Overridden" +) + type ConfigProperty struct { Id int `json:"id"` Name string `json:"name"` ConfigState ConfigState `json:"configState"` Type bean.ResourceType `json:"type"` - Overridden bool `json:"overridden"` - Global bool `json:"global"` + ConfigStage ConfigStage `json:"configStage"` } func NewConfigProperty() *ConfigProperty { @@ -31,16 +38,7 @@ func NewConfigProperty() *ConfigProperty { } func (r *ConfigProperty) IsConfigPropertyGlobal() bool { - return r.Global -} - -func (r *ConfigProperty) SetConfigProperty(Name string, ConfigState ConfigState, Type bean.ResourceType, Overridden bool, Global bool) *ConfigProperty { - r.Name = Name - r.ConfigState = ConfigState - r.Type = Type - r.Overridden = Overridden - r.Global = Global - return r + return r.ConfigStage == Inheriting } type ConfigDataResponse struct { @@ -73,28 +71,14 @@ func (r *ConfigProperty) GetIdentifier() ConfigPropertyIdentifier { } type DeploymentAndCmCsConfig struct { - Id int `json:"id ,omitempty"` - Name string `json:"name"` ResourceType bean.ResourceType `json:"resourceType"` Data json.RawMessage `json:"data"` - ConfigState ConfigState `json:"configState"` } func NewDeploymentAndCmCsConfig() *DeploymentAndCmCsConfig { return &DeploymentAndCmCsConfig{} } -func (r *DeploymentAndCmCsConfig) WithIdAndName(id int, name string) *DeploymentAndCmCsConfig { - r.Id = id - r.Name = name - return r -} - -func (r *DeploymentAndCmCsConfig) WithConfigState(configState ConfigState) *DeploymentAndCmCsConfig { - r.ConfigState = configState - return r -} - func (r *DeploymentAndCmCsConfig) WithResourceType(resourceType bean.ResourceType) *DeploymentAndCmCsConfig { r.ResourceType = resourceType return r @@ -106,23 +90,25 @@ func (r *DeploymentAndCmCsConfig) WithConfigData(data json.RawMessage) *Deployme } type DeploymentAndCmCsConfigDto struct { - AppId int `json:"appId"` - EnvironmentId int `json:"environmentId"` - ConfigData []*DeploymentAndCmCsConfig `json:"configData"` + DeploymentTemplate *DeploymentAndCmCsConfig `json:"deploymentTemplate"` + ConfigMapsData *DeploymentAndCmCsConfig `json:"configMapData"` + SecretsData *DeploymentAndCmCsConfig `json:"secretsData"` } func NewDeploymentAndCmCsConfigDto() *DeploymentAndCmCsConfigDto { - return &DeploymentAndCmCsConfigDto{ConfigData: make([]*DeploymentAndCmCsConfig, 0)} + return &DeploymentAndCmCsConfigDto{} } -func (r *DeploymentAndCmCsConfigDto) WithConfigData(configData []*DeploymentAndCmCsConfig) *DeploymentAndCmCsConfigDto { - r.ConfigData = configData +func (r *DeploymentAndCmCsConfigDto) WithDeploymentTemplateData(data *DeploymentAndCmCsConfig) *DeploymentAndCmCsConfigDto { + r.DeploymentTemplate = data return r } - -func (r *DeploymentAndCmCsConfigDto) WithAppAndEnvIdId(appId, envId int) *DeploymentAndCmCsConfigDto { - r.AppId = appId - r.EnvironmentId = envId +func (r *DeploymentAndCmCsConfigDto) WithConfigMapData(data *DeploymentAndCmCsConfig) *DeploymentAndCmCsConfigDto { + r.ConfigMapsData = data + return r +} +func (r *DeploymentAndCmCsConfigDto) WithSecretData(data *DeploymentAndCmCsConfig) *DeploymentAndCmCsConfigDto { + r.SecretsData = data return r } @@ -131,8 +117,11 @@ type ConfigDataQueryParams struct { EnvName string ConfigType string IdentifierId int + PipelineId int // req for fetching previous deployments data ResourceName string ResourceType string + ResourceId int + UserId int32 } func (r *ConfigDataQueryParams) IsResourceTypeSecret() bool { @@ -154,15 +143,6 @@ func (r *ConfigDataQueryParams) IsRequestMadeForOneResource() bool { return len(r.ResourceName) > 0 && len(r.ResourceType) > 0 } -func (r *ConfigDataQueryParams) WithConfigDataQueryParams(appName, envName, configType, resourceName, resourceType string, identifierId int) { - r.AppName = appName - r.EnvName = envName - r.ConfigType = configType - r.ResourceName = resourceName - r.IdentifierId = identifierId - r.ResourceType = resourceType -} - const ( InvalidConfigTypeErr = "invalid config type provided, please send a valid config type" ) From 9fbd056dfc8ecc1ebacf8d4b1916853c0e54263d Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 31 Jul 2024 17:31:16 +0530 Subject: [PATCH 34/45] code review comment incorp -1 --- .../DeploymentConfigurationRestHandler.go | 71 +++++-------------- internal/sql/repository/app/AppRepository.go | 15 ++++ .../repository/EnvironmentRepository.go | 13 ++++ .../DeploymentConfigurationService.go | 10 ++- pkg/configDiff/bean/bean.go | 25 ++++--- pkg/pipeline/ConfigMapService.go | 29 +++----- pkg/pipeline/bean/ConfigMapBean.go | 4 ++ 7 files changed, 76 insertions(+), 91 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 240d7d8a08f..25d0646f9c1 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -8,10 +8,10 @@ import ( "github.com/devtron-labs/devtron/pkg/configDiff" "github.com/devtron-labs/devtron/pkg/configDiff/bean" "github.com/devtron-labs/devtron/util/rbac" + "github.com/gorilla/schema" "go.uber.org/zap" "gopkg.in/go-playground/validator.v9" "net/http" - "strconv" ) type DeploymentConfigurationRestHandler interface { @@ -85,17 +85,15 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp return } - if len(configDataQueryParams.AppName) > 0 { - //RBAC START - token := r.Header.Get(common.TokenHeaderKey) - object := handler.enforcerUtil.GetAppRBACName(configDataQueryParams.AppName) - ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, casbin.ActionGet) - if !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), nil, http.StatusForbidden) - return - } - //RBAC END + //RBAC START + token := r.Header.Get(common.TokenHeaderKey) + object := handler.enforcerUtil.GetAppRBACName(configDataQueryParams.AppName) + ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, casbin.ActionGet) + if !ok { + common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), nil, http.StatusForbidden) + return } + //RBAC END res, err := handler.deploymentConfigurationService.GetAllConfigData(r.Context(), configDataQueryParams) if err != nil { @@ -108,50 +106,13 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp func getConfigDataQueryParams(r *http.Request) (*bean.ConfigDataQueryParams, error) { v := r.URL.Query() - - var identifierId int - var pipelineId int - var resourceId int - var err error - - appName := v.Get("appName") - envName := v.Get("envName") - configType := v.Get("configType") - identifierIdStr := v.Get("identifierId") - - if len(identifierIdStr) > 0 { - identifierId, err = strconv.Atoi(identifierIdStr) - if err != nil { - return nil, err - } - } - - pipelineIdStr := v.Get("pipelineId") - - if len(pipelineIdStr) > 0 { - pipelineId, err = strconv.Atoi(pipelineIdStr) - if err != nil { - return nil, err - } + var decoder = schema.NewDecoder() + decoder.IgnoreUnknownKeys(true) + queryParams := bean.ConfigDataQueryParams{} + err := decoder.Decode(&queryParams, v) + if err != nil { + return nil, err } - resourceName := v.Get("resourceName") - resourceType := v.Get("resourceType") - resourceIdStr := v.Get("resourceId") - if len(resourceIdStr) > 0 { - resourceId, err = strconv.Atoi(resourceIdStr) - if err != nil { - return nil, err - } - } - return &bean.ConfigDataQueryParams{ - AppName: appName, - EnvName: envName, - ConfigType: configType, - IdentifierId: identifierId, - ResourceName: resourceName, - ResourceType: resourceType, - PipelineId: pipelineId, - ResourceId: resourceId, - }, nil + return &queryParams, nil } diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index fef6630bfd7..76d526f1744 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -50,6 +50,8 @@ type AppRepository interface { UpdateWithTxn(app *App, tx *pg.Tx) error SetDescription(id int, description string, userId int32) error FindActiveByName(appName string) (pipelineGroup *App, err error) + FindAppIdByName(appName string) (int, error) + FindJobByDisplayName(appName string) (pipelineGroup *App, err error) FindActiveListByName(appName string) ([]*App, error) FindById(id int) (pipelineGroup *App, err error) @@ -137,6 +139,19 @@ func (repo AppRepositoryImpl) FindActiveByName(appName string) (*App, error) { // there is only single active app will be present in db with a same name. return pipelineGroup, err } +func (repo AppRepositoryImpl) FindAppIdByName(appName string) (int, error) { + app := &App{} + err := repo.dbConnection. + Model(app). + Column("app.id"). + Where("app_name = ?", appName). + Where("active = ?", true). + Order("id DESC").Limit(1). + Select() + // there is only single active app will be present in db with a same name. + return app.Id, err +} + func (repo AppRepositoryImpl) FindJobByDisplayName(appName string) (*App, error) { pipelineGroup := &App{} err := repo.dbConnection. diff --git a/pkg/cluster/repository/EnvironmentRepository.go b/pkg/cluster/repository/EnvironmentRepository.go index c1864ca2864..048aa83ae76 100644 --- a/pkg/cluster/repository/EnvironmentRepository.go +++ b/pkg/cluster/repository/EnvironmentRepository.go @@ -61,6 +61,7 @@ type EnvironmentRepository interface { FindById(id int) (*Environment, error) Update(mappings *Environment) error FindByName(name string) (*Environment, error) + FindIdByName(name string) (int, error) FindByIdentifier(identifier string) (*Environment, error) FindByNameOrIdentifier(name string, identifier string) (*Environment, error) FindByEnvNameOrIdentifierOrNamespace(clusterId int, envName string, identifier string, namespace string) (*Environment, error) @@ -159,6 +160,18 @@ func (repositoryImpl EnvironmentRepositoryImpl) FindByName(name string) (*Enviro return environment, err } +func (repositoryImpl EnvironmentRepositoryImpl) FindIdByName(name string) (int, error) { + environment := &Environment{} + err := repositoryImpl.dbConnection. + Model(environment). + Column("environment.id"). + Where("environment_name = ?", name). + Where("active = ?", true). + Limit(1). + Select() + return environment.Id, err +} + func (repositoryImpl EnvironmentRepositoryImpl) FindByIdentifier(identifier string) (*Environment, error) { environment := &Environment{} err := repositoryImpl.dbConnection. diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 15d74a584ad..360de7f8b35 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -89,19 +89,17 @@ func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Con var envId int var appId int if configDataQueryParams.IsEnvNameProvided() { - envModel, err := impl.environmentRepository.FindByName(configDataQueryParams.EnvName) + envId, err = impl.environmentRepository.FindIdByName(configDataQueryParams.EnvName) if err != nil { impl.logger.Errorw("GetAllConfigData, error in getting environment model by envName", "envName", configDataQueryParams.EnvName, "err", err) return nil, err } - envId = envModel.Id } - appModel, err := impl.appRepository.FindActiveByName(configDataQueryParams.AppName) + appId, err = impl.appRepository.FindAppIdByName(configDataQueryParams.AppName) if err != nil { impl.logger.Errorw("GetAllConfigData, error in getting app model by appName", "appName", configDataQueryParams.AppName, "err", err) return nil, err } - appId = appModel.Id configDataDto := &bean2.DeploymentAndCmCsConfigDto{} switch configDataQueryParams.ConfigType { @@ -259,7 +257,7 @@ func (impl *DeploymentConfigurationServiceImpl) getSecretConfigResponse(resource if envId > 0 { return impl.configMapService.CSEnvironmentFetchForEdit(resourceName, resourceId, appId, envId) } - return impl.configMapService.CSGlobalFetchForEditUsingAppId(resourceName, appId) + return impl.configMapService.ConfigGlobalFetchEditUsingAppId(resourceName, appId, bean.CS) } if envId > 0 { @@ -273,7 +271,7 @@ func (impl *DeploymentConfigurationServiceImpl) getConfigMapResponse(resourceNam if envId > 0 { return impl.configMapService.CMEnvironmentFetchForEdit(resourceName, resourceId, appId, envId) } - return impl.configMapService.CMGlobalFetchForEditUsingAppId(resourceName, appId) + return impl.configMapService.ConfigGlobalFetchEditUsingAppId(resourceName, appId, bean.CM) } if envId > 0 { diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 460d12b267f..b87ce14b683 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -113,17 +113,20 @@ func (r *DeploymentAndCmCsConfigDto) WithSecretData(data *DeploymentAndCmCsConfi } type ConfigDataQueryParams struct { - AppName string - EnvName string - ConfigType string - IdentifierId int - PipelineId int // req for fetching previous deployments data - ResourceName string - ResourceType string - ResourceId int - UserId int32 -} - + AppName string `schema:"appName"` + EnvName string `schema:"envName"` + ConfigType string `schema:"configType"` + IdentifierId int `schema:"identifierId"` + PipelineId int `schema:"pipelineId"` // req for fetching previous deployments data + ResourceName string `schema:"resourceName"` + ResourceType string `schema:"resourceType"` + ResourceId int `schema:"resourceId"` + UserId int32 `schema:"-"` +} + +// FilterCriteria []string `schema:"filterCriteria"` +// OffSet int `schema:"offSet"` +// Limit int `schema:"limit"` func (r *ConfigDataQueryParams) IsResourceTypeSecret() bool { return r.ResourceType == v1.Secret } diff --git a/pkg/pipeline/ConfigMapService.go b/pkg/pipeline/ConfigMapService.go index bb2d7334d7d..2501c813968 100644 --- a/pkg/pipeline/ConfigMapService.go +++ b/pkg/pipeline/ConfigMapService.go @@ -56,7 +56,7 @@ type ConfigMapService interface { CMEnvironmentFetch(appId int, envId int) (*bean.ConfigDataRequest, error) CMGlobalFetchForEdit(name string, id int) (*bean.ConfigDataRequest, error) CMEnvironmentFetchForEdit(name string, id int, appId int, envId int) (*bean.ConfigDataRequest, error) - CMGlobalFetchForEditUsingAppId(name string, appId int) (*bean.ConfigDataRequest, error) + ConfigGlobalFetchEditUsingAppId(name string, appId int, resourceType bean.ResourceType) (*bean.ConfigDataRequest, error) CSGlobalAddUpdate(configMapRequest *bean.ConfigDataRequest) (*bean.ConfigDataRequest, error) CSGlobalFetch(appId int) (*bean.ConfigDataRequest, error) @@ -73,7 +73,6 @@ type ConfigMapService interface { CSGlobalDeleteByAppId(name string, appId int, userId int32) (bool, error) CSEnvironmentDeleteByAppIdAndEnvId(name string, appId int, envId int, userId int32) (bool, error) - CSGlobalFetchForEditUsingAppId(name string, appId int) (*bean.ConfigDataRequest, error) CSGlobalFetchForEdit(name string, id int) (*bean.ConfigDataRequest, error) CSEnvironmentFetchForEdit(name string, id int, appId int, envId int) (*bean.ConfigDataRequest, error) ConfigSecretGlobalBulkPatch(bulkPatchRequest *bean.BulkPatchRequest) (*bean.BulkPatchRequest, error) @@ -2013,27 +2012,19 @@ func (impl ConfigMapServiceImpl) FetchCmCsNamesAppAndEnvLevel(appId int, envId i return cMCSNamesAppLevel, cMCSNamesEnvLevel, nil } -func (impl ConfigMapServiceImpl) CMGlobalFetchForEditUsingAppId(name string, appId int) (*bean.ConfigDataRequest, error) { - configDataRequest, err := impl.CMGlobalFetch(appId) - if err != nil { - return nil, err +func (impl ConfigMapServiceImpl) ConfigGlobalFetchEditUsingAppId(name string, appId int, resourceType bean.ResourceType) (*bean.ConfigDataRequest, error) { + var fetchGlobalConfigFunc func(int) (*bean.ConfigDataRequest, error) + if resourceType == bean.CS { + fetchGlobalConfigFunc = impl.CSGlobalFetch + } else if resourceType == bean.CM { + fetchGlobalConfigFunc = impl.CMGlobalFetch } - var configs []*bean.ConfigData - for _, configData := range configDataRequest.ConfigData { - if configData.Name == name { - configs = append(configs, configData) - } - } - configDataRequest.ConfigData = configs - return configDataRequest, nil -} - -func (impl ConfigMapServiceImpl) CSGlobalFetchForEditUsingAppId(name string, appId int) (*bean.ConfigDataRequest, error) { - configDataRequest, err := impl.CSGlobalFetch(appId) + configDataRequest, err := fetchGlobalConfigFunc(appId) if err != nil { + impl.logger.Errorw("error in fetching global cm using app id ", "cmName", name, "appId", appId, "err", err) return nil, err } - var configs []*bean.ConfigData + configs := make([]*bean.ConfigData, 0, len(configDataRequest.ConfigData)) for _, configData := range configDataRequest.ConfigData { if configData.Name == name { configs = append(configs, configData) diff --git a/pkg/pipeline/bean/ConfigMapBean.go b/pkg/pipeline/bean/ConfigMapBean.go index e66fc8dfa03..2f572bd6058 100644 --- a/pkg/pipeline/bean/ConfigMapBean.go +++ b/pkg/pipeline/bean/ConfigMapBean.go @@ -121,3 +121,7 @@ const ( CS ResourceType = "Secret" DeploymentTemplate ResourceType = "Deployment Template" ) + +func (r ResourceType) ToString() string { + return string(r) +} From 6f23f3050bc30f0491206fa670e71377e6756786 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 31 Jul 2024 18:25:06 +0530 Subject: [PATCH 35/45] code review comment incorp -2 --- .../DeploymentConfigurationRestHandler.go | 19 +++ pkg/pipeline/ConfigMapService.go | 142 ------------------ util/rbac/EnforcerUtil.go | 18 +++ wire_gen.go | 2 +- 4 files changed, 38 insertions(+), 143 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 25d0646f9c1..e29ceeef4a6 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -24,18 +24,21 @@ type DeploymentConfigurationRestHandlerImpl struct { validator *validator.Validate enforcerUtil rbac.EnforcerUtil deploymentConfigurationService configDiff.DeploymentConfigurationService + enforcer casbin.Enforcer } func NewDeploymentConfigurationRestHandlerImpl(logger *zap.SugaredLogger, userAuthService user.UserService, enforcerUtil rbac.EnforcerUtil, deploymentConfigurationService configDiff.DeploymentConfigurationService, + enforcer casbin.Enforcer, ) *DeploymentConfigurationRestHandlerImpl { return &DeploymentConfigurationRestHandlerImpl{ logger: logger, userAuthService: userAuthService, enforcerUtil: enforcerUtil, deploymentConfigurationService: deploymentConfigurationService, + enforcer: enforcer, } } @@ -100,10 +103,26 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp handler.logger.Errorw("service err, GetAllConfigData ", "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return + } + appAdminUser := handler.enforceForAppAndEnv(configDataQueryParams.AppName, configDataQueryParams.EnvName, token, casbin.ActionUpdate) + if !appAdminUser { + } common.WriteJsonResp(w, nil, res, http.StatusOK) } +func (handler *DeploymentConfigurationRestHandlerImpl) enforceForAppAndEnv(appName, envName string, token string, action string) bool { + object := handler.enforcerUtil.GetAppRBACNameByAppName(appName) + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, action, object); !ok { + return false + } + + object = handler.enforcerUtil.GetEnvRBACNameByAppAndEnvName(appName, envName) + if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, action, object); !ok { + return false + } + return true +} func getConfigDataQueryParams(r *http.Request) (*bean.ConfigDataQueryParams, error) { v := r.URL.Query() var decoder = schema.NewDecoder() diff --git a/pkg/pipeline/ConfigMapService.go b/pkg/pipeline/ConfigMapService.go index 2501c813968..56edec6e4a9 100644 --- a/pkg/pipeline/ConfigMapService.go +++ b/pkg/pipeline/ConfigMapService.go @@ -488,8 +488,6 @@ func (impl ConfigMapServiceImpl) CMEnvironmentFetch(appId int, envId int) (*bean if configDataRequest.ConfigData == nil { list := []*bean.ConfigData{} configDataRequest.ConfigData = list - } else { - //configDataRequest.ConfigData = configMapGlobalList.ConfigData } return configDataRequest, nil @@ -634,68 +632,15 @@ func (impl ConfigMapServiceImpl) CSGlobalFetch(appId int) (*bean.ConfigDataReque configDataRequest := &bean.ConfigDataRequest{} configDataRequest.Id = configMapGlobal.Id configDataRequest.AppId = appId - //configDataRequest.ConfigData = configMapGlobalList.ConfigData for _, item := range configMapGlobalList.ConfigData { item.Global = true configDataRequest.ConfigData = append(configDataRequest.ConfigData, item) } - //removing actual values - var configs []*bean.ConfigData - for _, item := range configDataRequest.ConfigData { - resultMap := make(map[string]string) - resultMapFinal := make(map[string]string) - - if item.Data != nil { - err = json.Unmarshal(item.Data, &resultMap) - if err != nil { - impl.logger.Warnw("unmarshal failed: ", "error", err) - configs = append(configs, item) - continue - } - for k := range resultMap { - resultMapFinal[k] = "" - } - resultByte, err := json.Marshal(resultMapFinal) - if err != nil { - impl.logger.Errorw("error while marshaling request ", "err", err) - return nil, err - } - item.Data = resultByte - } - - var externalSecret []bean.ExternalSecret - if item.ExternalSecret != nil && len(item.ExternalSecret) > 0 { - for _, es := range item.ExternalSecret { - externalSecret = append(externalSecret, bean.ExternalSecret{Key: es.Key, Name: es.Name, Property: es.Property, IsBinary: es.IsBinary}) - } - } - item.ExternalSecret = externalSecret - - var esoData []bean.ESOData - if len(item.ESOSecretData.EsoData) > 0 { - for _, data := range item.ESOSecretData.EsoData { - esoData = append(esoData, bean.ESOData{Key: data.Key, SecretKey: data.SecretKey, Property: data.Property}) - } - } - - esoSecretData := bean.ESOSecretData{ - SecretStore: item.ESOSecretData.SecretStore, - SecretStoreRef: item.ESOSecretData.SecretStoreRef, - EsoData: esoData, - RefreshInterval: item.ESOSecretData.RefreshInterval, - } - item.ESOSecretData = esoSecretData - configs = append(configs, item) - } - configDataRequest.ConfigData = configs - if configDataRequest.ConfigData == nil { list := []*bean.ConfigData{} configDataRequest.ConfigData = list - } else { - //configDataRequest.ConfigData = configMapGlobalList.ConfigData } return configDataRequest, nil @@ -922,93 +867,6 @@ func (impl ConfigMapServiceImpl) CSEnvironmentFetch(appId int, envId int) (*bean } } - //removing actual values - var configs []*bean.ConfigData - for _, item := range configDataRequest.ConfigData { - - if item.Data != nil { - resultMap := make(map[string]string) - resultMapFinal := make(map[string]string) - err = json.Unmarshal(item.Data, &resultMap) - if err != nil { - impl.logger.Warnw("unmarshal failed: ", "error", err) - //item.Data = []byte("[]") - configs = append(configs, item) - continue - //return nil, err - } - for k := range resultMap { - resultMapFinal[k] = "" - } - var resultByte []byte - if resultMapFinal != nil && len(resultMapFinal) > 0 { - resultByte, err = json.Marshal(resultMapFinal) - if err != nil { - impl.logger.Errorw("error while marshaling request ", "err", err) - return nil, err - } - } - item.Data = resultByte - } - - var externalSecret []bean.ExternalSecret - if item.ExternalSecret != nil && len(item.ExternalSecret) > 0 { - for _, es := range item.ExternalSecret { - externalSecret = append(externalSecret, bean.ExternalSecret{Key: es.Key, Name: es.Name, Property: es.Property, IsBinary: es.IsBinary}) - } - } - item.ExternalSecret = externalSecret - - var esoData []bean.ESOData - if len(item.ESOSecretData.EsoData) > 0 { - for _, data := range item.ESOSecretData.EsoData { - esoData = append(esoData, bean.ESOData{Key: data.Key, SecretKey: data.SecretKey, Property: data.Property}) - } - } - - esoSecretData := bean.ESOSecretData{ - SecretStore: item.ESOSecretData.SecretStore, - SecretStoreRef: item.ESOSecretData.SecretStoreRef, - EsoData: esoData, - RefreshInterval: item.ESOSecretData.RefreshInterval, - } - item.ESOSecretData = esoSecretData - - if item.DefaultData != nil { - resultMap := make(map[string]string) - resultMapFinal := make(map[string]string) - err = json.Unmarshal(item.DefaultData, &resultMap) - if err != nil { - impl.logger.Warnw("unmarshal failed: ", "error", err) - //item.Data = []byte("[]") - configs = append(configs, item) - continue - //return nil, err - } - for k := range resultMap { - resultMapFinal[k] = "" - } - resultByte, err := json.Marshal(resultMapFinal) - if err != nil { - impl.logger.Errorw("error while marshaling request ", "err", err) - return nil, err - } - item.DefaultData = resultByte - } - - if item.DefaultExternalSecret != nil { - var externalSecret []bean.ExternalSecret - if item.DefaultExternalSecret != nil && len(item.DefaultExternalSecret) > 0 { - for _, es := range item.DefaultExternalSecret { - externalSecret = append(externalSecret, bean.ExternalSecret{Key: es.Key, Name: es.Name, Property: es.Property, IsBinary: es.IsBinary}) - } - } - item.DefaultExternalSecret = externalSecret - } - configs = append(configs, item) - } - configDataRequest.ConfigData = configs - if configDataRequest.ConfigData == nil { list := []*bean.ConfigData{} configDataRequest.ConfigData = list diff --git a/util/rbac/EnforcerUtil.go b/util/rbac/EnforcerUtil.go index bb646e70355..77f3b952130 100644 --- a/util/rbac/EnforcerUtil.go +++ b/util/rbac/EnforcerUtil.go @@ -76,6 +76,8 @@ type EnforcerUtil interface { CheckAppRbacForAppOrJob(token, resourceName, action string) bool CheckAppRbacForAppOrJobInBulk(token, action string, rbacObjects []string, appType helper.AppType) map[string]bool GetRbacObjectsByEnvIdsAndAppIdBatch(appIdToEnvIds map[int][]int) map[int]map[int]string + GetEnvRBACNameByAppAndEnvName(appName, envName string) string + GetAppRBACNameByAppName(appName string) string } type EnforcerUtilImpl struct { @@ -764,3 +766,19 @@ func (impl EnforcerUtilImpl) GetRbacObjectsByEnvIdsAndAppIdBatch(appIdToEnvIds m } return objects } + +func (impl EnforcerUtilImpl) GetAppRBACNameByAppName(appName string) string { + application, err := impl.appRepo.FindAppAndProjectByAppName(appName) + if err != nil { + return fmt.Sprintf("%s/%s", "", "") + } + return fmt.Sprintf("%s/%s", application.Team.Name, application.AppName) +} + +func (impl EnforcerUtilImpl) GetEnvRBACNameByAppAndEnvName(appName, envName string) string { + env, err := impl.environmentRepository.FindByName(envName) + if err != nil { + return fmt.Sprintf("%s/%s", "", appName) + } + return fmt.Sprintf("%s/%s", env.EnvironmentIdentifier, appName) +} diff --git a/wire_gen.go b/wire_gen.go index 87a719f2c31..cb5845a3660 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -934,7 +934,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - deploymentConfigurationRestHandlerImpl := restHandler.NewDeploymentConfigurationRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerUtilImpl, deploymentConfigurationServiceImpl) + deploymentConfigurationRestHandlerImpl := restHandler.NewDeploymentConfigurationRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerUtilImpl, deploymentConfigurationServiceImpl, enforcerImpl) deploymentConfigurationRouterImpl := router.NewDeploymentConfigurationRouter(deploymentConfigurationRestHandlerImpl) infraConfigRestHandlerImpl := infraConfig2.NewInfraConfigRestHandlerImpl(sugaredLogger, infraConfigServiceImpl, userServiceImpl, enforcerImpl, enforcerUtilImpl, validate) infraConfigRouterImpl := infraConfig2.NewInfraProfileRouterImpl(infraConfigRestHandlerImpl) From cf5ce0bb5d1a104f63f75227f9f7373b740c7c45 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 31 Jul 2024 19:15:58 +0530 Subject: [PATCH 36/45] code review comment incorp -3 --- api/restHandler/DeploymentConfigurationRestHandler.go | 4 +--- pkg/configDiff/bean/bean.go | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index e29ceeef4a6..a29776a6b65 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -104,10 +104,8 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - appAdminUser := handler.enforceForAppAndEnv(configDataQueryParams.AppName, configDataQueryParams.EnvName, token, casbin.ActionUpdate) - if !appAdminUser { + res.IsAppAdmin = handler.enforceForAppAndEnv(configDataQueryParams.AppName, configDataQueryParams.EnvName, token, casbin.ActionUpdate) - } common.WriteJsonResp(w, nil, res, http.StatusOK) } diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index b87ce14b683..61caf434c23 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -93,6 +93,7 @@ type DeploymentAndCmCsConfigDto struct { DeploymentTemplate *DeploymentAndCmCsConfig `json:"deploymentTemplate"` ConfigMapsData *DeploymentAndCmCsConfig `json:"configMapData"` SecretsData *DeploymentAndCmCsConfig `json:"secretsData"` + IsAppAdmin bool `json:"isAppAdmin"` } func NewDeploymentAndCmCsConfigDto() *DeploymentAndCmCsConfigDto { From 7fd7eee2ac663bfba1b168d6fa0842f03cae1170 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 6 Aug 2024 15:38:00 +0530 Subject: [PATCH 37/45] small fix in plugin --- pkg/plugin/bean/bean.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/plugin/bean/bean.go b/pkg/plugin/bean/bean.go index 9e152272e37..55424f3caac 100644 --- a/pkg/plugin/bean/bean.go +++ b/pkg/plugin/bean/bean.go @@ -171,6 +171,7 @@ func (r *PluginsVersionDetail) SetMinimalPluginsVersionDetail(pluginVersionMetad r.Description = pluginVersionMetadata.Description r.Version = pluginVersionMetadata.PluginVersion r.IsLatest = pluginVersionMetadata.IsLatest + r.DocLink = pluginVersionMetadata.DocLink return r } From 779e913181fc5636193bc762b5ec24faada4870c Mon Sep 17 00:00:00 2001 From: Prakash Date: Thu, 15 Aug 2024 23:23:54 +0530 Subject: [PATCH 38/45] migration number changes (#5692) --- ...an_plugin.down.sql => 270_improved_image_scan_plugin.down.sql} | 0 ...e_scan_plugin.up.sql => 270_improved_image_scan_plugin.up.sql} | 0 ...rent_metadata.down.sql => 271_plugin_parent_metadata.down.sql} | 0 ...n_parent_metadata.up.sql => 271_plugin_parent_metadata.up.sql} | 0 ...lugin_metadata.down.sql => 272_alter_plugin_metadata.down.sql} | 0 ...er_plugin_metadata.up.sql => 272_alter_plugin_metadata.up.sql} | 0 ...ls_support_in_git.down.sql => 273_tls_support_in_git.down.sql} | 0 ...72_tls_support_in_git.up.sql => 273_tls_support_in_git.up.sql} | 0 ..._system_controller.down.sql => 274_system_controller.down.sql} | 0 ...{273_system_controller.up.sql => 274_system_controller.up.sql} | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename scripts/sql/{274_improved_image_scan_plugin.down.sql => 270_improved_image_scan_plugin.down.sql} (100%) rename scripts/sql/{274_improved_image_scan_plugin.up.sql => 270_improved_image_scan_plugin.up.sql} (100%) rename scripts/sql/{270_plugin_parent_metadata.down.sql => 271_plugin_parent_metadata.down.sql} (100%) rename scripts/sql/{270_plugin_parent_metadata.up.sql => 271_plugin_parent_metadata.up.sql} (100%) rename scripts/sql/{271_alter_plugin_metadata.down.sql => 272_alter_plugin_metadata.down.sql} (100%) rename scripts/sql/{271_alter_plugin_metadata.up.sql => 272_alter_plugin_metadata.up.sql} (100%) rename scripts/sql/{272_tls_support_in_git.down.sql => 273_tls_support_in_git.down.sql} (100%) rename scripts/sql/{272_tls_support_in_git.up.sql => 273_tls_support_in_git.up.sql} (100%) rename scripts/sql/{273_system_controller.down.sql => 274_system_controller.down.sql} (100%) rename scripts/sql/{273_system_controller.up.sql => 274_system_controller.up.sql} (100%) diff --git a/scripts/sql/274_improved_image_scan_plugin.down.sql b/scripts/sql/270_improved_image_scan_plugin.down.sql similarity index 100% rename from scripts/sql/274_improved_image_scan_plugin.down.sql rename to scripts/sql/270_improved_image_scan_plugin.down.sql diff --git a/scripts/sql/274_improved_image_scan_plugin.up.sql b/scripts/sql/270_improved_image_scan_plugin.up.sql similarity index 100% rename from scripts/sql/274_improved_image_scan_plugin.up.sql rename to scripts/sql/270_improved_image_scan_plugin.up.sql diff --git a/scripts/sql/270_plugin_parent_metadata.down.sql b/scripts/sql/271_plugin_parent_metadata.down.sql similarity index 100% rename from scripts/sql/270_plugin_parent_metadata.down.sql rename to scripts/sql/271_plugin_parent_metadata.down.sql diff --git a/scripts/sql/270_plugin_parent_metadata.up.sql b/scripts/sql/271_plugin_parent_metadata.up.sql similarity index 100% rename from scripts/sql/270_plugin_parent_metadata.up.sql rename to scripts/sql/271_plugin_parent_metadata.up.sql diff --git a/scripts/sql/271_alter_plugin_metadata.down.sql b/scripts/sql/272_alter_plugin_metadata.down.sql similarity index 100% rename from scripts/sql/271_alter_plugin_metadata.down.sql rename to scripts/sql/272_alter_plugin_metadata.down.sql diff --git a/scripts/sql/271_alter_plugin_metadata.up.sql b/scripts/sql/272_alter_plugin_metadata.up.sql similarity index 100% rename from scripts/sql/271_alter_plugin_metadata.up.sql rename to scripts/sql/272_alter_plugin_metadata.up.sql diff --git a/scripts/sql/272_tls_support_in_git.down.sql b/scripts/sql/273_tls_support_in_git.down.sql similarity index 100% rename from scripts/sql/272_tls_support_in_git.down.sql rename to scripts/sql/273_tls_support_in_git.down.sql diff --git a/scripts/sql/272_tls_support_in_git.up.sql b/scripts/sql/273_tls_support_in_git.up.sql similarity index 100% rename from scripts/sql/272_tls_support_in_git.up.sql rename to scripts/sql/273_tls_support_in_git.up.sql diff --git a/scripts/sql/273_system_controller.down.sql b/scripts/sql/274_system_controller.down.sql similarity index 100% rename from scripts/sql/273_system_controller.down.sql rename to scripts/sql/274_system_controller.down.sql diff --git a/scripts/sql/273_system_controller.up.sql b/scripts/sql/274_system_controller.up.sql similarity index 100% rename from scripts/sql/273_system_controller.up.sql rename to scripts/sql/274_system_controller.up.sql From 728de28a3baf7d865c54697bf237e0511b81ad46 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 16 Aug 2024 18:55:53 +0530 Subject: [PATCH 39/45] main sync --- cmd/external-app/wire_gen.go | 6 ++--- wire_gen.go | 44 +++++++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index c3a4512ff85..23dbbb83014 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -199,8 +199,8 @@ func InitializeApp() (*App, error) { userAuthServiceImpl := user.NewUserAuthServiceImpl(userAuthRepositoryImpl, sessionManager, loginService, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, userServiceImpl) teamServiceImpl := team.NewTeamServiceImpl(sugaredLogger, teamRepositoryImpl, userAuthServiceImpl) clusterRepositoryImpl := repository2.NewClusterRepositoryImpl(db, sugaredLogger) - v := informer.NewGlobalMapClusterNamespace() - k8sInformerFactoryImpl := informer.NewK8sInformerFactoryImpl(sugaredLogger, v, k8sServiceImpl) + syncMap := informer.NewGlobalMapClusterNamespace() + k8sInformerFactoryImpl := informer.NewK8sInformerFactoryImpl(sugaredLogger, syncMap, k8sServiceImpl) clusterServiceImpl := cluster.NewClusterServiceImpl(clusterRepositoryImpl, sugaredLogger, k8sServiceImpl, k8sInformerFactoryImpl, userAuthRepositoryImpl, userRepositoryImpl, roleGroupRepositoryImpl) appStatusRepositoryImpl := appStatus.NewAppStatusRepositoryImpl(db, sugaredLogger) environmentRepositoryImpl := repository2.NewEnvironmentRepositoryImpl(db, sugaredLogger, appStatusRepositoryImpl) diff --git a/wire_gen.go b/wire_gen.go index 56b7d5a1939..0bc013a9cf0 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -31,6 +31,7 @@ import ( deployment2 "github.com/devtron-labs/devtron/api/deployment" devtronResource2 "github.com/devtron-labs/devtron/api/devtronResource" externalLink2 "github.com/devtron-labs/devtron/api/externalLink" + fluxApplication2 "github.com/devtron-labs/devtron/api/fluxApplication" client3 "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/api/helm-app/service" @@ -118,6 +119,7 @@ import ( service3 "github.com/devtron-labs/devtron/pkg/appStore/values/service" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" "github.com/devtron-labs/devtron/pkg/argoApplication" + "github.com/devtron-labs/devtron/pkg/argoRepositoryCreds" "github.com/devtron-labs/devtron/pkg/asyncProvider" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authentication" @@ -135,6 +137,7 @@ import ( "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess" "github.com/devtron-labs/devtron/pkg/commonService" + "github.com/devtron-labs/devtron/pkg/configDiff" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" @@ -162,6 +165,7 @@ import ( "github.com/devtron-labs/devtron/pkg/eventProcessor/in" "github.com/devtron-labs/devtron/pkg/eventProcessor/out" "github.com/devtron-labs/devtron/pkg/externalLink" + "github.com/devtron-labs/devtron/pkg/fluxApplication" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/genericNotes" repository6 "github.com/devtron-labs/devtron/pkg/genericNotes/repository" @@ -245,7 +249,7 @@ func InitializeApp() (*App, error) { attributesServiceImpl := attributes.NewAttributesServiceImpl(sugaredLogger, attributesRepositoryImpl) grafanaClientImpl := grafana.NewGrafanaClientImpl(sugaredLogger, httpClient, grafanaClientConfig, attributesServiceImpl) installedAppRepositoryImpl := repository3.NewInstalledAppRepositoryImpl(sugaredLogger, db) - runtimeConfig, err := client.GetRuntimeConfig() + runtimeConfig, err := k8s.GetRuntimeConfig() if err != nil { return nil, err } @@ -264,15 +268,19 @@ func InitializeApp() (*App, error) { return nil, err } serviceClientImpl := cluster.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) - v := informer.NewGlobalMapClusterNamespace() - k8sInformerFactoryImpl := informer.NewK8sInformerFactoryImpl(sugaredLogger, v, runtimeConfig, k8sServiceImpl) + syncMap := informer.NewGlobalMapClusterNamespace() + k8sInformerFactoryImpl := informer.NewK8sInformerFactoryImpl(sugaredLogger, syncMap, k8sServiceImpl) defaultAuthPolicyRepositoryImpl := repository4.NewDefaultAuthPolicyRepositoryImpl(db, sugaredLogger) defaultAuthRoleRepositoryImpl := repository4.NewDefaultAuthRoleRepositoryImpl(db, sugaredLogger) userAuthRepositoryImpl := repository4.NewUserAuthRepositoryImpl(db, sugaredLogger, defaultAuthPolicyRepositoryImpl, defaultAuthRoleRepositoryImpl) userRepositoryImpl := repository4.NewUserRepositoryImpl(db, sugaredLogger) roleGroupRepositoryImpl := repository4.NewRoleGroupRepositoryImpl(db, sugaredLogger) gitOpsConfigRepositoryImpl := repository2.NewGitOpsConfigRepositoryImpl(sugaredLogger, db) - k8sClient, err := client.NewK8sClient(runtimeConfig) + clientRuntimeConfig, err := client.GetRuntimeConfig() + if err != nil { + return nil, err + } + k8sClient, err := client.NewK8sClient(clientRuntimeConfig) if err != nil { return nil, err } @@ -356,7 +364,8 @@ func InitializeApp() (*App, error) { dockerArtifactStoreRepositoryImpl := repository5.NewDockerArtifactStoreRepositoryImpl(db) dockerRegistryIpsConfigRepositoryImpl := repository5.NewDockerRegistryIpsConfigRepositoryImpl(db) ociRegistryConfigRepositoryImpl := repository5.NewOCIRegistryConfigRepositoryImpl(db) - dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, helmAppServiceImpl, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl, ociRegistryConfigRepositoryImpl) + repositorySecretImpl := argoRepositoryCreds.NewRepositorySecret(sugaredLogger, k8sServiceImpl, clusterServiceImplExtended, acdAuthConfig) + dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, helmAppServiceImpl, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl, ociRegistryConfigRepositoryImpl, repositorySecretImpl) deleteServiceExtendedImpl := delete2.NewDeleteServiceExtendedImpl(sugaredLogger, teamServiceImpl, clusterServiceImplExtended, environmentServiceImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, chartRepositoryServiceImpl, installedAppRepositoryImpl, dockerRegistryConfigImpl, dockerArtifactStoreRepositoryImpl) versionServiceImpl := argocdServer.NewVersionServiceImpl(sugaredLogger, argoCDConnectionManagerImpl) moduleActionAuditLogRepositoryImpl := module.NewModuleActionAuditLogRepositoryImpl(db) @@ -386,7 +395,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - argoApplicationServiceImpl := argoApplication.NewArgoApplicationServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, argoUserServiceImpl, helmAppServiceImpl) + argoApplicationServiceImpl := argoApplication.NewArgoApplicationServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl) k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sServiceImpl, clusterServiceImplExtended, argoApplicationServiceImpl) environmentRestHandlerImpl := cluster3.NewEnvironmentRestHandlerImpl(environmentServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceExtendedImpl, k8sServiceImpl, k8sCommonServiceImpl) environmentRouterImpl := cluster3.NewEnvironmentRouterImpl(environmentRestHandlerImpl) @@ -584,7 +593,7 @@ func InitializeApp() (*App, error) { ciTemplateHistoryRepositoryImpl := repository12.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) - ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl) + ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl) ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, transactionUtilImpl) deploymentGroupRepositoryImpl := repository2.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryRepositoryImpl := repository12.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) @@ -701,11 +710,12 @@ func InitializeApp() (*App, error) { ephemeralContainersRepositoryImpl := repository.NewEphemeralContainersRepositoryImpl(db, transactionUtilImpl) ephemeralContainerServiceImpl := cluster2.NewEphemeralContainerServiceImpl(ephemeralContainersRepositoryImpl, sugaredLogger) terminalSessionHandlerImpl := terminal.NewTerminalSessionHandlerImpl(environmentServiceImpl, clusterServiceImplExtended, sugaredLogger, k8sServiceImpl, ephemeralContainerServiceImpl, argoApplicationServiceImpl) - k8sApplicationServiceImpl, err := application2.NewK8sApplicationServiceImpl(sugaredLogger, clusterServiceImplExtended, pumpImpl, helmAppServiceImpl, k8sServiceImpl, acdAuthConfig, k8sResourceHistoryServiceImpl, k8sCommonServiceImpl, terminalSessionHandlerImpl, ephemeralContainerServiceImpl, ephemeralContainersRepositoryImpl, argoApplicationServiceImpl) + fluxApplicationServiceImpl := fluxApplication.NewFluxApplicationServiceImpl(sugaredLogger, helmAppServiceImpl, clusterServiceImplExtended, helmAppClientImpl, pumpImpl) + k8sApplicationServiceImpl, err := application2.NewK8sApplicationServiceImpl(sugaredLogger, clusterServiceImplExtended, pumpImpl, helmAppServiceImpl, k8sServiceImpl, acdAuthConfig, k8sResourceHistoryServiceImpl, k8sCommonServiceImpl, terminalSessionHandlerImpl, ephemeralContainerServiceImpl, ephemeralContainersRepositoryImpl, argoApplicationServiceImpl, fluxApplicationServiceImpl) if err != nil { return nil, err } - installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, applicationServiceClientImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl, deploymentConfigServiceImpl) + installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, applicationServiceClientImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl, deploymentConfigServiceImpl, ociRegistryConfigRepositoryImpl) chartGroupEntriesRepositoryImpl := repository17.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) chartGroupReposotoryImpl := repository17.NewChartGroupReposotoryImpl(db, sugaredLogger) chartGroupDeploymentRepositoryImpl := repository17.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) @@ -714,7 +724,7 @@ func InitializeApp() (*App, error) { clusterInstalledAppsRepositoryImpl := repository3.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) appStoreValuesServiceImpl := service3.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, chartTemplateServiceImpl) - fullModeDeploymentServiceImpl := deployment.NewFullModeDeploymentServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig, gitOperationServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, environmentRepositoryImpl, deploymentConfigServiceImpl, chartTemplateServiceImpl) + fullModeDeploymentServiceImpl := deployment.NewFullModeDeploymentServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig, gitOperationServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, environmentRepositoryImpl, deploymentConfigServiceImpl, chartTemplateServiceImpl, repositorySecretImpl) appStoreValidatorImpl := service4.NewAppAppStoreValidatorImpl(sugaredLogger) appStoreDeploymentDBServiceImpl := service4.NewAppStoreDeploymentDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, environmentServiceImpl, clusterServiceImplExtended, installedAppVersionHistoryRepositoryImpl, environmentVariables, gitOpsConfigReadServiceImpl, deploymentTypeOverrideServiceImpl, fullModeDeploymentServiceImpl, appStoreValidatorImpl, installedAppDBServiceImpl, deploymentConfigServiceImpl) eaModeDeploymentServiceImpl := EAMode.NewEAModeDeploymentServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, ociRegistryConfigRepositoryImpl) @@ -854,9 +864,9 @@ func InitializeApp() (*App, error) { appRouterImpl := app3.NewAppRouterImpl(appFilteringRouterImpl, appListingRouterImpl, appInfoRouterImpl, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, pipelineHistoryRouterImpl, pipelineStatusRouterImpl, appWorkflowRouterImpl, devtronAppAutoCompleteRouterImpl, appWorkflowRestHandlerImpl, appListingRestHandlerImpl, appFilteringRestHandlerImpl) coreAppRestHandlerImpl := restHandler.NewCoreAppRestHandlerImpl(sugaredLogger, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, appCrudOperationServiceImpl, pipelineBuilderImpl, gitRegistryConfigImpl, chartServiceImpl, configMapServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, gitProviderRepositoryImpl, appWorkflowRepositoryImpl, environmentRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, teamServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, ciPipelineRepositoryImpl) coreAppRouterImpl := router.NewCoreAppRouterImpl(coreAppRestHandlerImpl) - helmAppRestHandlerImpl := client3.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImplExtended, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, installedAppDBServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig) + helmAppRestHandlerImpl := client3.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImplExtended, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, installedAppDBServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig, fluxApplicationServiceImpl, argoApplicationServiceImpl) helmAppRouterImpl := client3.NewHelmAppRouterImpl(helmAppRestHandlerImpl) - k8sApplicationRestHandlerImpl := application3.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilImpl, helmAppServiceImpl, userServiceImpl, k8sCommonServiceImpl, validate, environmentVariables) + k8sApplicationRestHandlerImpl := application3.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilImpl, helmAppServiceImpl, userServiceImpl, k8sCommonServiceImpl, validate, environmentVariables, fluxApplicationServiceImpl, argoApplicationServiceImpl) k8sApplicationRouterImpl := application3.NewK8sApplicationRouterImpl(k8sApplicationRestHandlerImpl) pProfRestHandlerImpl := restHandler.NewPProfRestHandler(userServiceImpl, enforcerImpl) pProfRouterImpl := router.NewPProfRouter(sugaredLogger, pProfRestHandlerImpl) @@ -933,6 +943,12 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } + deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl) + if err != nil { + return nil, err + } + deploymentConfigurationRestHandlerImpl := restHandler.NewDeploymentConfigurationRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerUtilImpl, deploymentConfigurationServiceImpl, enforcerImpl) + deploymentConfigurationRouterImpl := router.NewDeploymentConfigurationRouter(deploymentConfigurationRestHandlerImpl) infraConfigRestHandlerImpl := infraConfig2.NewInfraConfigRestHandlerImpl(sugaredLogger, infraConfigServiceImpl, userServiceImpl, enforcerImpl, enforcerUtilImpl, validate) infraConfigRouterImpl := infraConfig2.NewInfraProfileRouterImpl(infraConfigRestHandlerImpl) argoApplicationRestHandlerImpl := argoApplication2.NewArgoApplicationRestHandlerImpl(argoApplicationServiceImpl, sugaredLogger, enforcerImpl) @@ -942,7 +958,9 @@ func InitializeApp() (*App, error) { historyRestHandlerImpl := devtronResource2.NewHistoryRestHandlerImpl(sugaredLogger, enforcerImpl, deploymentHistoryServiceImpl, apiReqDecoderServiceImpl, enforcerUtilImpl) historyRouterImpl := devtronResource2.NewHistoryRouterImpl(historyRestHandlerImpl) devtronResourceRouterImpl := devtronResource2.NewDevtronResourceRouterImpl(historyRouterImpl) - muxRouter := router.NewMuxRouter(sugaredLogger, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl, infraConfigRouterImpl, argoApplicationRouterImpl, devtronResourceRouterImpl) + fluxApplicationRestHandlerImpl := fluxApplication2.NewFluxApplicationRestHandlerImpl(fluxApplicationServiceImpl, sugaredLogger, enforcerImpl) + fluxApplicationRouterImpl := fluxApplication2.NewFluxApplicationRouterImpl(fluxApplicationRestHandlerImpl) + muxRouter := router.NewMuxRouter(sugaredLogger, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl, deploymentConfigurationRouterImpl, infraConfigRouterImpl, argoApplicationRouterImpl, devtronResourceRouterImpl, fluxApplicationRouterImpl) loggingMiddlewareImpl := util4.NewLoggingMiddlewareImpl(userServiceImpl) cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) From 2fa3c2e5acea526f371ddfe8c62d66d7905efc21 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 19 Aug 2024 17:02:16 +0530 Subject: [PATCH 40/45] minor fix --- pkg/configDiff/bean/bean.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 61caf434c23..2113ea81a65 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -1,9 +1,9 @@ package bean +import "C" import ( "encoding/json" "fmt" - v1 "github.com/devtron-labs/devtron/pkg/apis/devtron/v1" "github.com/devtron-labs/devtron/pkg/pipeline/bean" ) @@ -129,11 +129,11 @@ type ConfigDataQueryParams struct { // OffSet int `schema:"offSet"` // Limit int `schema:"limit"` func (r *ConfigDataQueryParams) IsResourceTypeSecret() bool { - return r.ResourceType == v1.Secret + return r.ResourceType == bean.CS.ToString() } func (r *ConfigDataQueryParams) IsResourceTypeConfigMap() bool { - return r.ResourceType == v1.ConfigMap + return r.ResourceType == bean.CM.ToString() } func (r *ConfigDataQueryParams) IsEnvNameProvided() bool { From 4f04d6b5eee794f768f025cf58dea0e9ce5f80f0 Mon Sep 17 00:00:00 2001 From: Prakash Date: Tue, 20 Aug 2024 12:35:33 +0530 Subject: [PATCH 41/45] refrain from checkin autoscalingCheckBeforeTrigger for virt clus (#5696) --- pkg/deployment/manifest/ManifestCreationService.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/deployment/manifest/ManifestCreationService.go b/pkg/deployment/manifest/ManifestCreationService.go index 86641dfeac9..7547e067298 100644 --- a/pkg/deployment/manifest/ManifestCreationService.go +++ b/pkg/deployment/manifest/ManifestCreationService.go @@ -275,10 +275,12 @@ func (impl *ManifestCreationServiceImpl) GetValuesOverrideForTrigger(overrideReq // error is not returned as it's not blocking for deployment process // blocking deployments based on this use case can vary for user to user } - mergedValues, err = impl.autoscalingCheckBeforeTrigger(newCtx, appName, envOverride.Namespace, mergedValues, overrideRequest) - if err != nil { - impl.logger.Errorw("error in autoscaling check before trigger", "pipelineId", overrideRequest.PipelineId, "err", err) - return valuesOverrideResponse, err + if !envOverride.Environment.IsVirtualEnvironment { + mergedValues, err = impl.autoscalingCheckBeforeTrigger(newCtx, appName, envOverride.Namespace, mergedValues, overrideRequest) + if err != nil { + impl.logger.Errorw("error in autoscaling check before trigger", "pipelineId", overrideRequest.PipelineId, "err", err) + return valuesOverrideResponse, err + } } // handle image pull secret if access given mergedValues, err = impl.dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment(newCtx, envOverride.Environment, artifact, pipeline.CiPipelineId, mergedValues) From 2e58e77959a458671d0b70a7bab4041a9f5c4894 Mon Sep 17 00:00:00 2001 From: Prakash Date: Tue, 20 Aug 2024 12:50:03 +0530 Subject: [PATCH 42/45] fix: Decode secret fix on add update oss (#5695) * ValidateEncodedDataByDecoding in case add or update secret * wire fix from main * minor refactor * comment --- cmd/external-app/wire_gen.go | 4 +-- pkg/pipeline/ConfigMapService.go | 42 ++++++++++++++++++++++++++------ util/encoding-utils.go | 10 ++++++++ wire_gen.go | 2 +- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index c3a4512ff85..72533cab867 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -199,8 +199,8 @@ func InitializeApp() (*App, error) { userAuthServiceImpl := user.NewUserAuthServiceImpl(userAuthRepositoryImpl, sessionManager, loginService, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, userServiceImpl) teamServiceImpl := team.NewTeamServiceImpl(sugaredLogger, teamRepositoryImpl, userAuthServiceImpl) clusterRepositoryImpl := repository2.NewClusterRepositoryImpl(db, sugaredLogger) - v := informer.NewGlobalMapClusterNamespace() - k8sInformerFactoryImpl := informer.NewK8sInformerFactoryImpl(sugaredLogger, v, k8sServiceImpl) + syncMap := informer.NewGlobalMapClusterNamespace() + k8sInformerFactoryImpl := informer.NewK8sInformerFactoryImpl(sugaredLogger, syncMap, k8sServiceImpl) clusterServiceImpl := cluster.NewClusterServiceImpl(clusterRepositoryImpl, sugaredLogger, k8sServiceImpl, k8sInformerFactoryImpl, userAuthRepositoryImpl, userRepositoryImpl, roleGroupRepositoryImpl) appStatusRepositoryImpl := appStatus.NewAppStatusRepositoryImpl(db, sugaredLogger) environmentRepositoryImpl := repository2.NewEnvironmentRepositoryImpl(db, sugaredLogger, appStatusRepositoryImpl) diff --git a/pkg/pipeline/ConfigMapService.go b/pkg/pipeline/ConfigMapService.go index 77e996dcce5..49c08772a15 100644 --- a/pkg/pipeline/ConfigMapService.go +++ b/pkg/pipeline/ConfigMapService.go @@ -34,7 +34,9 @@ import ( util2 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.uber.org/zap" + "net/http" "regexp" + "strconv" "time" ) @@ -504,12 +506,19 @@ func (impl ConfigMapServiceImpl) CSGlobalAddUpdate(configMapRequest *bean.Config return nil, fmt.Errorf("invalid request multiple config found for add or update") } configData := configMapRequest.ConfigData[0] + // validating config/secret data at service layer since this func is consumed in multiple flows, hence preventing code duplication valid, err := impl.validateConfigData(configData) if err != nil && !valid { impl.logger.Errorw("error in validating", "error", err) return configMapRequest, err } + valid, err = impl.validateConfigDataForSecretsOnly(configData) + if err != nil && !valid { + impl.logger.Errorw("error in validating secrets only data", "error", err) + return configMapRequest, err + } + valid, err = impl.validateExternalSecretChartCompatibility(configMapRequest.AppId, configMapRequest.EnvironmentId, configData) if err != nil && !valid { impl.logger.Errorw("error in validating", "error", err) @@ -704,11 +713,17 @@ func (impl ConfigMapServiceImpl) CSEnvironmentAddUpdate(configMapRequest *bean.C } configData := configMapRequest.ConfigData[0] + // validating config/secret data at service layer since this func is consumed in multiple flows, hence preventing code duplication valid, err := impl.validateConfigData(configData) if err != nil && !valid { impl.logger.Errorw("error in validating", "error", err) return configMapRequest, err } + valid, err = impl.validateConfigDataForSecretsOnly(configData) + if err != nil && !valid { + impl.logger.Errorw("error in validating secrets only data", "error", err) + return configMapRequest, err + } valid, err = impl.validateExternalSecretChartCompatibility(configMapRequest.AppId, configMapRequest.EnvironmentId, configData) if err != nil && !valid { @@ -795,13 +810,6 @@ func (impl ConfigMapServiceImpl) CSEnvironmentAddUpdate(configMapRequest *bean.C } configMapRequest.Id = configMap.Id } - //VARIABLE_MAPPING_UPDATE - //sl := bean.SecretsList{} - //data, err := sl.GetTransformedDataForSecretList(model.SecretData, util2.DecodeSecret) - //if err != nil { - // return nil, err - //} - //err = impl.extractAndMapVariables(data, model.Id, repository5.EntityTypeSecretEnvLevel, configMapRequest.UserId) err = impl.scopedVariableManager.CreateVariableMappingsForSecretEnv(model) if err != nil { return nil, err @@ -1545,6 +1553,26 @@ func (impl ConfigMapServiceImpl) validateConfigData(configData *bean.ConfigData) return true, nil } +func (impl ConfigMapServiceImpl) validateConfigDataForSecretsOnly(configData *bean.ConfigData) (bool, error) { + + // check encoding in base64 for secret data + if len(configData.Data) > 0 { + dataMap := make(map[string]string) + err := json.Unmarshal(configData.Data, &dataMap) + if err != nil { + impl.logger.Errorw("error while unmarshalling secret data ", "error", err) + return false, err + } + err = util2.ValidateEncodedDataByDecoding(dataMap) + if err != nil { + impl.logger.Errorw("error in decoding secret data", "error", err) + return false, util.NewApiError().WithHttpStatusCode(http.StatusUnprocessableEntity).WithCode(strconv.Itoa(http.StatusUnprocessableEntity)). + WithUserMessage("error in decoding data, make sure the secret data is encoded properly") + } + } + return true, nil +} + func (impl ConfigMapServiceImpl) updateConfigData(configData *bean.ConfigData, syncRequest *bean.BulkPatchRequest) (*bean.ConfigData, error) { dataMap := make(map[string]string) var updatedData json.RawMessage diff --git a/util/encoding-utils.go b/util/encoding-utils.go index 88064a26bd5..82837c229cb 100644 --- a/util/encoding-utils.go +++ b/util/encoding-utils.go @@ -53,3 +53,13 @@ func GetDecodedAndEncodedData(data json.RawMessage, transformer SecretTransformM } return marshal, nil } + +func ValidateEncodedDataByDecoding(dataMap map[string]string) error { + for _, value := range dataMap { + _, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return err + } + } + return nil +} diff --git a/wire_gen.go b/wire_gen.go index b1ef4f1028b..d83a360fd09 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -714,7 +714,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, applicationServiceClientImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl, deploymentConfigServiceImpl) + installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, applicationServiceClientImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl, deploymentConfigServiceImpl, ociRegistryConfigRepositoryImpl) chartGroupEntriesRepositoryImpl := repository17.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) chartGroupReposotoryImpl := repository17.NewChartGroupReposotoryImpl(db, sugaredLogger) chartGroupDeploymentRepositoryImpl := repository17.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) From bf2351544aa57137980cbc3b6561b88d21e359b6 Mon Sep 17 00:00:00 2001 From: Prakash Date: Tue, 20 Aug 2024 12:55:08 +0530 Subject: [PATCH 43/45] saving pco concurrency case handled (#5688) --- .../sql/repository/chartConfig/PipelineOverrideRepository.go | 5 +++++ pkg/deployment/manifest/ManifestCreationService.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/sql/repository/chartConfig/PipelineOverrideRepository.go b/internal/sql/repository/chartConfig/PipelineOverrideRepository.go index aba3d7554c6..db8476d9132 100644 --- a/internal/sql/repository/chartConfig/PipelineOverrideRepository.go +++ b/internal/sql/repository/chartConfig/PipelineOverrideRepository.go @@ -58,6 +58,7 @@ type PipelineConfigOverrideMetadata struct { type PipelineOverrideRepository interface { Save(*PipelineOverride) error + Update(pipelineOverride *PipelineOverride) error UpdateStatusByRequestIdentifier(requestId string, newStatus models.ChartStatus) (int, error) GetLatestConfigByRequestIdentifier(requestIdentifier string) (pipelineOverride *PipelineOverride, err error) GetLatestConfigByEnvironmentConfigOverrideId(envConfigOverrideId int) (pipelineOverride *PipelineOverride, err error) @@ -85,6 +86,10 @@ func (impl PipelineOverrideRepositoryImpl) Save(pipelineOverride *PipelineOverri return impl.dbConnection.Insert(pipelineOverride) } +func (impl PipelineOverrideRepositoryImpl) Update(pipelineOverride *PipelineOverride) error { + return impl.dbConnection.Update(pipelineOverride) +} + func (impl PipelineOverrideRepositoryImpl) UpdatePipelineMergedValues(ctx context.Context, tx *pg.Tx, id int, pipelineMergedValues string, userId int32) error { _, span := otel.Tracer("orchestrator").Start(ctx, "PipelineOverrideRepositoryImpl.UpdatePipelineMergedValues") defer span.End() diff --git a/pkg/deployment/manifest/ManifestCreationService.go b/pkg/deployment/manifest/ManifestCreationService.go index 7547e067298..9b2cc199274 100644 --- a/pkg/deployment/manifest/ManifestCreationService.go +++ b/pkg/deployment/manifest/ManifestCreationService.go @@ -808,7 +808,7 @@ func (impl *ManifestCreationServiceImpl) checkAndFixDuplicateReleaseNo(override return err } override.PipelineReleaseCounter = currentReleaseNo + 1 - err = impl.pipelineOverrideRepository.Save(override) + err = impl.pipelineOverrideRepository.Update(override) if err != nil { return err } From 694831c209a2a28883b299cb8c073ef2441dddca Mon Sep 17 00:00:00 2001 From: Prakash Date: Wed, 21 Aug 2024 13:30:04 +0530 Subject: [PATCH 44/45] fix: script for pipelineStageStepVariable, making input value and default_value text from varchar255 (#5701) * script for pipelineStageStepVariable, making input value and default_value text from varchar255 * erro log fix --- pkg/deployment/gitOps/git/GitOpsHelper.go | 1 + pkg/deployment/gitOps/git/GitServiceGithub.go | 1 + pkg/deployment/gitOps/git/commandManager/GitCliManager.go | 3 +++ pkg/deployment/gitOps/git/commandManager/GoGitSdkManager.go | 3 +++ scripts/sql/276_alter_pipeline_stage_step_variable.down.sql | 3 +++ scripts/sql/276_alter_pipeline_stage_step_variable.up.sql | 2 ++ 6 files changed, 13 insertions(+) create mode 100644 scripts/sql/276_alter_pipeline_stage_step_variable.down.sql create mode 100644 scripts/sql/276_alter_pipeline_stage_step_variable.up.sql diff --git a/pkg/deployment/gitOps/git/GitOpsHelper.go b/pkg/deployment/gitOps/git/GitOpsHelper.go index 2d204ef8a0d..bc7add2ff92 100644 --- a/pkg/deployment/gitOps/git/GitOpsHelper.go +++ b/pkg/deployment/gitOps/git/GitOpsHelper.go @@ -89,6 +89,7 @@ func (impl *GitOpsHelper) Clone(url, targetDir string) (clonedDir string, err er } } if errMsg != "" { + impl.logger.Errorw("error in git fetch command", "errMsg", errMsg, "err", err) return "", fmt.Errorf(errMsg) } return clonedDir, nil diff --git a/pkg/deployment/gitOps/git/GitServiceGithub.go b/pkg/deployment/gitOps/git/GitServiceGithub.go index 589d79c2ac7..b48d8b5ab43 100644 --- a/pkg/deployment/gitOps/git/GitServiceGithub.go +++ b/pkg/deployment/gitOps/git/GitServiceGithub.go @@ -259,6 +259,7 @@ func (impl GitHubClient) GetRepoUrl(config *bean2.GitOpsConfigDto) (repoUrl stri ctx := context.Background() repo, _, err := impl.client.Repositories.Get(ctx, impl.org, config.GitRepoName) if err != nil { + impl.logger.Errorw("error in getting repo url by repo name", "org", impl.org, "gitRepoName", config.GitRepoName, "err", err) return "", err } return *repo.CloneURL, nil diff --git a/pkg/deployment/gitOps/git/commandManager/GitCliManager.go b/pkg/deployment/gitOps/git/commandManager/GitCliManager.go index 0501f67cb34..b5b3a3a146d 100644 --- a/pkg/deployment/gitOps/git/commandManager/GitCliManager.go +++ b/pkg/deployment/gitOps/git/commandManager/GitCliManager.go @@ -77,6 +77,9 @@ func (impl *GitCliManagerImpl) Pull(ctx GitContext, repoRoot string) (err error) return err } response, errMsg, err := impl.PullCli(ctx, repoRoot, "origin/master") + if err != nil { + impl.logger.Errorw("error in git pull from cli", "errMsg", errMsg, "err", err) + } if strings.Contains(response, "already up-to-date") || strings.Contains(errMsg, "already up-to-date") { err = nil diff --git a/pkg/deployment/gitOps/git/commandManager/GoGitSdkManager.go b/pkg/deployment/gitOps/git/commandManager/GoGitSdkManager.go index 950a0bf4516..3d70a73c311 100644 --- a/pkg/deployment/gitOps/git/commandManager/GoGitSdkManager.go +++ b/pkg/deployment/gitOps/git/commandManager/GoGitSdkManager.go @@ -56,6 +56,9 @@ func (impl GoGitSDKManagerImpl) Pull(ctx GitContext, repoRoot string) (err error } err = workTree.PullContext(ctx, pullOptions) + if err != nil { + impl.logger.Errorw("error in git pull from go-git", "err", err) + } if err != nil && err.Error() == "already up-to-date" { err = nil return nil diff --git a/scripts/sql/276_alter_pipeline_stage_step_variable.down.sql b/scripts/sql/276_alter_pipeline_stage_step_variable.down.sql new file mode 100644 index 00000000000..bad0b4c4928 --- /dev/null +++ b/scripts/sql/276_alter_pipeline_stage_step_variable.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE pipeline_stage_step_variable ALTER COLUMN default_value TYPE VARCHAR(255); +ALTER TABLE pipeline_stage_step_variable ALTER COLUMN value TYPE VARCHAR(255); + diff --git a/scripts/sql/276_alter_pipeline_stage_step_variable.up.sql b/scripts/sql/276_alter_pipeline_stage_step_variable.up.sql new file mode 100644 index 00000000000..cbcf6515c90 --- /dev/null +++ b/scripts/sql/276_alter_pipeline_stage_step_variable.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE pipeline_stage_step_variable ALTER COLUMN value TYPE text; +ALTER TABLE pipeline_stage_step_variable ALTER COLUMN default_value TYPE text; From 3843077a74416176f5acdef0e3af91adc97a3220 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 21 Aug 2024 15:33:35 +0530 Subject: [PATCH 45/45] scipt number change --- .../{276_scan_policies.down.sql => 277_scan_policies.down.sql} | 0 .../sql/{276_scan_policies.up.sql => 277_scan_policies.up.sql} | 0 ...{277_rbac_role_audit.down.sql => 278_rbac_role_audit.down.sql} | 0 .../{277_rbac_role_audit.up.sql => 278_rbac_role_audit.up.sql} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename scripts/sql/{276_scan_policies.down.sql => 277_scan_policies.down.sql} (100%) rename scripts/sql/{276_scan_policies.up.sql => 277_scan_policies.up.sql} (100%) rename scripts/sql/{277_rbac_role_audit.down.sql => 278_rbac_role_audit.down.sql} (100%) rename scripts/sql/{277_rbac_role_audit.up.sql => 278_rbac_role_audit.up.sql} (100%) diff --git a/scripts/sql/276_scan_policies.down.sql b/scripts/sql/277_scan_policies.down.sql similarity index 100% rename from scripts/sql/276_scan_policies.down.sql rename to scripts/sql/277_scan_policies.down.sql diff --git a/scripts/sql/276_scan_policies.up.sql b/scripts/sql/277_scan_policies.up.sql similarity index 100% rename from scripts/sql/276_scan_policies.up.sql rename to scripts/sql/277_scan_policies.up.sql diff --git a/scripts/sql/277_rbac_role_audit.down.sql b/scripts/sql/278_rbac_role_audit.down.sql similarity index 100% rename from scripts/sql/277_rbac_role_audit.down.sql rename to scripts/sql/278_rbac_role_audit.down.sql diff --git a/scripts/sql/277_rbac_role_audit.up.sql b/scripts/sql/278_rbac_role_audit.up.sql similarity index 100% rename from scripts/sql/277_rbac_role_audit.up.sql rename to scripts/sql/278_rbac_role_audit.up.sql