Stress tests for DAG runtime modifications (openvinotoolkit#382)

TODO:
-> add other API calls -> getModelStatus, getModelMetadata

*Made:
-> checkResponse
-> stringError
as test_utils functions.
*Rename function checkResponse to checkDummyResponse
*Remove unnecessary includes
*Return to old wait multiplier factor
*Make loadConfig & loadConfigurationWithoutConfigFile protected
to enable using them in test without explicit sleep.
*Minor log improvements in model management.

JIRA:CVS-41787

src/BUILD

@@ -176,6 +176,7 @@ cc_test(
         "test/ensemble_tests.cpp",
         "test/ensemble_mapping_config_tests.cpp",
         "test/ensemble_metadata_test.cpp",
+        "test/ensemble_config_change_stress.cpp",
         "test/get_model_metadata_response_test.cpp",
         "test/get_pipeline_metadata_response_test.cpp",
         "test/get_model_metadata_signature_test.cpp",

src/dl_node.cpp

@@ -89,7 +89,7 @@ Status DLNode::setInputsForInference(InferenceEngine::InferRequest& infer_reques
         for (const auto& kv : this->inputBlobs) {
             std::string realModelInputName;
             if (!getRealInputName(kv.first, &realModelInputName).ok()) {
-                SPDLOG_WARN("DLNode::fetchResults (Node name {}); cannot find real model input name for ali{}", getName(), kv.first);
+                SPDLOG_WARN("DLNode::{} [Node name: {}]; cannot find real model input name for alias: {}", __FUNCTION__, getName(), kv.first);
                 return StatusCode::INTERNAL_ERROR;
             }
             infer_request.SetBlob(realModelInputName, kv.second);

src/entry_node.cpp

@@ -89,7 +89,6 @@ Status EntryNode::deserialize(const tensorflow::TensorProto& proto, InferenceEng
     }
 
     // description.setLayout();  // Layout info is stored in model instance. If we find out it is required, then need to be set right before inference.
-
     try {
         switch (proto.dtype()) {
         case tensorflow::DataType::DT_FLOAT:

src/modelmanager.cpp

@@ -442,6 +442,14 @@ void ModelManager::retireModelsRemovedFromConfigFile(const std::set<std::string>
     }
 }
 
+void ModelManager::updateConfigurationWithoutConfigFile() {
+    SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Checking if something changed with model versions");
+    for (auto& [name, config] : servedModelConfigs) {
+        reloadModelWithVersions(config);
+    }
+    pipelineFactory.revalidatePipelines(*this);
+}
+
 void ModelManager::watcher(std::future<void> exit) {
     SPDLOG_LOGGER_INFO(modelmanager_logger, "Started config watcher thread");
     int64_t lastTime;
@@ -456,10 +464,7 @@ void ModelManager::watcher(std::future<void> exit) {
             lastTime = statTime.st_ctime;
             loadConfig(configFilename);
         }
-        for (auto& [name, config] : servedModelConfigs) {
-            reloadModelWithVersions(config);
-        }
-        pipelineFactory.revalidatePipelines(*this);
+        updateConfigurationWithoutConfigFile();
         SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Watcher thread check cycle end");
     }
     SPDLOG_LOGGER_ERROR(modelmanager_logger, "Exited config watcher thread");
@@ -484,9 +489,9 @@ void ModelManager::getVersionsToChange(
     std::shared_ptr<model_versions_t>& versionsToRetireIn) {
     std::sort(requestedVersions.begin(), requestedVersions.end());
     model_versions_t registeredModelVersions;
-    SPDLOG_DEBUG("Currently registered versions count: {}", modelVersionsInstances.size());
+    SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Currently registered model: {} versions count: {}", newModelConfig.getName(), modelVersionsInstances.size());
     for (const auto& [version, versionInstance] : modelVersionsInstances) {
-        SPDLOG_DEBUG("version: {} state: {}", version, ovms::ModelVersionStateToString(versionInstance->getStatus().getState()));
+        SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {} version: {} state: {}", newModelConfig.getName(), version, ovms::ModelVersionStateToString(versionInstance->getStatus().getState()));
         registeredModelVersions.push_back(version);
     }
 
@@ -648,6 +653,7 @@ Status ModelManager::reloadModelVersions(std::shared_ptr<ovms::Model>& model, st
 }
 
 Status ModelManager::reloadModelWithVersions(ModelConfig& config) {
+    SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Started applying config changes to model: {}", config.getName());
     auto model = getModelIfExistCreateElse(config.getName());
     if (model->isAnyVersionSubscribed() && config.isDynamicParameterEnabled()) {
         SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested setting dynamic parameters for model {} but it is used in pipeline. Cannot reload model configuration.", config.getName());
@@ -661,11 +667,9 @@ Status ModelManager::reloadModelWithVersions(ModelConfig& config) {
         return blocking_status;
     }
     requestedVersions = config.getModelVersionPolicy()->filter(requestedVersions);
-
     std::shared_ptr<model_versions_t> versionsToStart;
     std::shared_ptr<model_versions_t> versionsToReload;
     std::shared_ptr<model_versions_t> versionsToRetire;
-
     // first reset custom loader name to empty string so that any changes to name can be captured
     model->resetCustomLoaderName();
 

src/modelmanager.hpp

@@ -62,13 +62,7 @@ class ModelManager {
      */
     ModelManager(const ModelManager&) = delete;
 
-    /**
-     * @brief Reads models from configuration file
-     * 
-     * @param jsonFilename configuration file
-     * @return Status 
-     */
-    Status loadConfig(const std::string& jsonFilename);
+    Status cleanupModelTmpFiles(ModelConfig& config);
     Status reloadModelVersions(std::shared_ptr<ovms::Model>& model, std::shared_ptr<FileSystem>& fs, ModelConfig& config, std::shared_ptr<model_versions_t>& versionsToReload);
     Status addModelVersions(std::shared_ptr<ovms::Model>& model, std::shared_ptr<FileSystem>& fs, ModelConfig& config, std::shared_ptr<model_versions_t>& versionsToStart);
     Status loadModelsConfig(rapidjson::Document& configJson, std::vector<ModelConfig>& gatedModelConfigs);
@@ -297,6 +291,20 @@ class ModelManager {
         std::shared_ptr<model_versions_t>& versionsToStartIn);
 
     static std::shared_ptr<FileSystem> getFilesystem(const std::string& basePath);
+
+protected:
+    /**
+     * @brief Reads models from configuration file
+     * 
+     * @param jsonFilename configuration file
+     * @return Status 
+     */
+    Status loadConfig(const std::string& jsonFilename);
+
+    /**
+     * @brief Updates OVMS configuration with cached configuration file. Will check for newly added model versions
+     */
+    void updateConfigurationWithoutConfigFile();
 };
 
 }  // namespace ovms

src/status.hpp

@@ -186,6 +186,8 @@ enum class StatusCode {
     CUSTOM_LOADER_NOT_PRESENT,
     CUSTOM_LOADER_INIT_FAILED,
     CUSTOM_LOADER_ERROR,
+
+    STATUS_CODE_END
 };
 
 class Status {