Update agent to match server behavior when staging job dependencies

Genie V3 downloads job dependencies and configurations to the root of the job folder. Whereas the agent was creating 'configurations' and 'dependencies' directories. Update the agent setup code to match the server setup code.
vitortiagozup · May 31, 2019 · d879a53 · d879a53
1 parent 90b1bb5
commit d879a53
Show file tree

Hide file tree

Showing 2 changed files with 48 additions and 8 deletions.
diff --git a/.../src/main/java/com/netflix/genie/agent/execution/statemachine/actions/SetUpJobAction.java b/.../src/main/java/com/netflix/genie/agent/execution/statemachine/actions/SetUpJobAction.java
@@ -275,8 +275,7 @@ private void createJobDirectoryStructure(
         // Make a list of all entity dirs
         final List<Path> entityDirectories = Lists.newArrayList(
             PathUtils.jobClusterDirectoryPath(jobDirectory, cluster.getId()),
-            PathUtils.jobCommandDirectoryPath(jobDirectory, command.getId()),
-            jobDirectory.toPath()
+            PathUtils.jobCommandDirectoryPath(jobDirectory, command.getId())
         );
 
         applications.stream()
@@ -342,9 +341,50 @@ private DownloadService.Manifest createDownloadManifest(
             PathUtils.jobCommandDirectoryPath(jobDirectory, commandId);
         addEntitiesFilesToManifest(commandDirectory, downloadManifestBuilder, command, setupFileUris);
 
-        // Job
-        final JobSpecification.ExecutionResource jobRequest = jobSpec.getJob();
-        addEntitiesFilesToManifest(jobDirectory.toPath(), downloadManifestBuilder, jobRequest, setupFileUris);
+        // Job (does not follow convention, downloads everything in the job root folder).
+        try {
+            final Path jobDirectoryPath = jobDirectory.toPath();
+            final ExecutionEnvironment jobExecEnvironment = jobSpec.getJob().getExecutionEnvironment();
+
+            if (jobExecEnvironment.getSetupFile().isPresent()) {
+                final URI setupFileUri = new URI(jobExecEnvironment.getSetupFile().get());
+                log.debug(
+                    "Adding setup file to download manifest: {} -> {}",
+                    setupFileUri,
+                    jobDirectoryPath
+                );
+                downloadManifestBuilder.addFileWithTargetDirectory(
+                    setupFileUri,
+                    jobDirectory
+                );
+                setupFileUris.add(setupFileUri);
+            }
+
+            for (final String dependencyUriString : jobExecEnvironment.getDependencies()) {
+                log.debug(
+                    "Adding dependency to download manifest: {} -> {}",
+                    dependencyUriString,
+                    jobDirectoryPath
+                );
+                downloadManifestBuilder.addFileWithTargetDirectory(
+                    new URI(dependencyUriString), jobDirectory
+                );
+            }
+
+            for (final String configUriString : jobExecEnvironment.getConfigs()) {
+                log.debug(
+                    "Adding config file to download manifest: {} -> {}",
+                    configUriString,
+                    jobDirectoryPath
+                );
+                downloadManifestBuilder.addFileWithTargetDirectory(
+                    new URI(configUriString),
+                    jobDirectory
+                );
+            }
+        } catch (final URISyntaxException e) {
+            throw new SetUpJobException("Failed to compose download manifest", e);
+        }
 
         // Build manifest
         final DownloadService.Manifest manifest = downloadManifestBuilder.build();

diff --git a/...t/groovy/com/netflix/genie/agent/execution/statemachine/actions/SetUpJobActionSpec.groovy b/...t/groovy/com/netflix/genie/agent/execution/statemachine/actions/SetUpJobActionSpec.groovy
@@ -260,7 +260,7 @@ class SetUpJobActionSpec extends Specification {
         envMap.get(JobConstants.GENIE_COMMAND_DIR_ENV_VAR) == jobDir.toString() + "/" + JobConstants.GENIE_PATH_VAR + "/" + JobConstants.COMMAND_PATH_VAR + "/" + command.getId()
         envMap.get(JobConstants.GENIE_CLUSTER_DIR_ENV_VAR) == jobDir.toString() + "/" + JobConstants.GENIE_PATH_VAR + "/" + JobConstants.CLUSTER_PATH_VAR + "/" + cluster.getId()
 
-        for (File entityDir : [jobDir, app1Dir, app2Dir, clusterDir, commandDir]) {
+        for (File entityDir : [app1Dir, app2Dir, clusterDir, commandDir]) {
             assert entityDir.exists()
             def confDir = new File(entityDir, JobConstants.CONFIG_FILE_PATH_PREFIX)
             assert confDir.exists()
@@ -324,13 +324,13 @@ class SetUpJobActionSpec extends Specification {
         1 * manifestBuilder.addFileWithTargetDirectory(setupFileUri, commandDir)
         1 * manifestBuilder.addFileWithTargetDirectory(setupFileUri, clusterDir)
         5 * manifest.getTargetLocation(setupFileUri) >> dummyFile
-        1 * manifestBuilder.addFileWithTargetDirectory(dependencyUri, new File(jobDir, JobConstants.DEPENDENCY_FILE_PATH_PREFIX))
+        1 * manifestBuilder.addFileWithTargetDirectory(dependencyUri, jobDir)
         1 * manifestBuilder.addFileWithTargetDirectory(dependencyUri, new File(app1Dir, JobConstants.DEPENDENCY_FILE_PATH_PREFIX))
         1 * manifestBuilder.addFileWithTargetDirectory(dependencyUri, new File(app2Dir, JobConstants.DEPENDENCY_FILE_PATH_PREFIX))
         1 * manifestBuilder.addFileWithTargetDirectory(dependencyUri, new File(commandDir, JobConstants.DEPENDENCY_FILE_PATH_PREFIX))
         1 * manifestBuilder.addFileWithTargetDirectory(dependencyUri, new File(clusterDir, JobConstants.DEPENDENCY_FILE_PATH_PREFIX))
         0 * manifest.getTargetLocation(dependencyUri)
-        1 * manifestBuilder.addFileWithTargetDirectory(configUri, new File(jobDir, JobConstants.CONFIG_FILE_PATH_PREFIX))
+        1 * manifestBuilder.addFileWithTargetDirectory(configUri, jobDir)
         1 * manifestBuilder.addFileWithTargetDirectory(configUri, new File(app1Dir, JobConstants.CONFIG_FILE_PATH_PREFIX))
         1 * manifestBuilder.addFileWithTargetDirectory(configUri, new File(app2Dir, JobConstants.CONFIG_FILE_PATH_PREFIX))
         1 * manifestBuilder.addFileWithTargetDirectory(configUri, new File(commandDir, JobConstants.CONFIG_FILE_PATH_PREFIX))