Skip to content

Commit

Permalink
Add runtime to V3 API Job Request model
Browse files Browse the repository at this point in the history
  • Loading branch information
@tgianos
tgianos committed May 9, 2022
1 parent 6275550 commit 8647973
Show file tree
Hide file tree
Showing 4 changed files with 156 additions and 40 deletions.
36 changes: 27 additions & 9 deletions ...ternal/src/main/java/com/netflix/genie/common/internal/dtos/converters/DtoConverters.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -489,15 +488,16 @@ public static JobRequest toV4JobRequest(
v3JobRequest.getApplications()
);

final Runtime runtime = v3JobRequest.getRuntime();
final JobEnvironmentRequest.Builder jobEnvironmentBuilder = new JobEnvironmentRequest.Builder();
final Optional<Integer> requestedCpu = v3JobRequest.getCpu();
final Optional<Integer> requestedMemory = v3JobRequest.getMemory();
if (requestedCpu.isPresent() || requestedMemory.isPresent()) {
final ComputeResources.Builder computeResourcesBuilder = new ComputeResources.Builder();
requestedCpu.ifPresent(computeResourcesBuilder::withCpu);
requestedMemory.ifPresent(memory -> computeResourcesBuilder.withMemoryMb(memory.longValue()));
jobEnvironmentBuilder.withRequestedComputeResources(computeResourcesBuilder.build());
}
jobEnvironmentBuilder.withRequestedComputeResources(toComputeResources(runtime.getResources()));
jobEnvironmentBuilder.withRequestedImages(
runtime
.getImages()
.entrySet()
.stream()
.collect(Collectors.toMap(Map.Entry::getKey, entry -> toImage(entry.getValue())))
);

final AgentConfigRequest.Builder agentConfigBuilder = new AgentConfigRequest
.Builder()
Expand Down Expand Up @@ -561,6 +561,24 @@ public static com.netflix.genie.common.dto.JobRequest toV3JobRequest(final JobRe
jobResources.getSetupFile().ifPresent(v3Builder::withSetupFile);

v4JobRequest.getRequestedAgentConfig().getTimeoutRequested().ifPresent(v3Builder::withTimeout);
final JobEnvironmentRequest jobEnvironmentRequest = v4JobRequest.getRequestedJobEnvironment();
v3Builder.withRuntime(
new Runtime.Builder()
.withResources(toV3RuntimeResources(jobEnvironmentRequest.getRequestedComputeResources()))
.withImages(
jobEnvironmentRequest
.getRequestedImages()
.entrySet()
.stream()
.collect(
Collectors.toMap(
Map.Entry::getKey,
entry -> toV3ContainerImage(entry.getValue())
)
)
)
.build()
);

return v3Builder.build();
}
Expand Down
66 changes: 59 additions & 7 deletions genie-common/src/main/java/com/netflix/genie/common/dto/JobRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,10 @@
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.Size;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

Expand Down Expand Up @@ -69,14 +71,17 @@ public class JobRequest extends ExecutionEnvironmentDTO {
@Email(message = "Must be a valid email address")
private final String email;
@Min(value = 1, message = "Must have at least 1 CPU")
@Deprecated
private final Integer cpu;
@Min(value = 1, message = "Must have at least 1 MB of memory. Preferably much more.")
@Deprecated
private final Integer memory;
@Min(value = 1, message = "The timeout must be at least 1 second, preferably much more.")
private final Integer timeout;
private final List<String> applications;
private final String grouping;
private final String groupingInstance;
private final Runtime runtime;

/**
* Constructor used by the builder build() method.
Expand All @@ -91,12 +96,20 @@ public class JobRequest extends ExecutionEnvironmentDTO {
this.group = builder.bGroup;
this.disableLogArchival = builder.bDisableLogArchival;
this.email = builder.bEmail;
this.cpu = builder.bCpu;
this.memory = builder.bMemory;
this.timeout = builder.bTimeout;
this.applications = ImmutableList.copyOf(builder.bApplications);
this.grouping = builder.bGrouping;
this.groupingInstance = builder.bGroupingInstance;
this.runtime = new Runtime.Builder()
.withResources(builder.bRuntimeResources.build())
.withImages(builder.bImages)
.build();

this.cpu = this.runtime.getResources().getCpu().orElse(null);
this.memory = this.runtime.getResources()
.getMemoryMb()
.map(Long::intValue)
.orElse(null);
}

/**
Expand Down Expand Up @@ -130,7 +143,10 @@ public Optional<String> getEmail() {
* Get the number of CPU's requested to run this job.
*
* @return The number of CPU's as an Optional
* @deprecated Use runtime instead
* @since 4.3.0
*/
@Deprecated
public Optional<Integer> getCpu() {
return Optional.ofNullable(this.cpu);
}
Expand All @@ -139,7 +155,10 @@ public Optional<Integer> getCpu() {
* Get the amount of memory (in MB) requested to run this job with.
*
* @return The amount of memory as an Optional
* @deprecated Use runtime instead
* @since 4.3.0
*/
@Deprecated
public Optional<Integer> getMemory() {
return Optional.ofNullable(this.memory);
}
Expand Down Expand Up @@ -188,11 +207,11 @@ public static class Builder extends ExecutionEnvironmentDTO.Builder<Builder> {
private String bGroup;
private boolean bDisableLogArchival;
private String bEmail;
private Integer bCpu;
private Integer bMemory;
private Integer bTimeout;
private String bGrouping;
private String bGroupingInstance;
private final RuntimeResources.Builder bRuntimeResources = new RuntimeResources.Builder();
private final Map<String, ContainerImage> bImages = new HashMap<>();

/**
* Constructor which has required fields.
Expand Down Expand Up @@ -365,20 +384,24 @@ public Builder withEmail(@Nullable final String email) {
*
* @param cpu The number of cpu's. Must be greater than 0.
* @return The builder
* @deprecated Use {@link #withRuntime(Runtime)} instead
*/
@Deprecated
public Builder withCpu(@Nullable final Integer cpu) {
this.bCpu = cpu;
this.bRuntimeResources.withCpu(cpu);
return this;
}

/**
* Set the amount of memory being requested to run the job..
* Set the amount of memory being requested to run the job.
*
* @param memory The amount of memory in terms of MB's. Must be greater than 0.
* @return The builder
* @deprecated Use {@link #withRuntime(Runtime)} instead
*/
@Deprecated
public Builder withMemory(@Nullable final Integer memory) {
this.bMemory = memory;
this.bRuntimeResources.withMemoryMb(memory == null ? null : memory.longValue());
return this;
}

Expand Down Expand Up @@ -431,6 +454,35 @@ public Builder withGroupingInstance(@Nullable final String groupingInstance) {
return this;
}

/**
* Set the requested job runtime environment details.
*
* @param runtime The {@link Runtime} information or {@literal null}
* @return The {@link Builder} instance
*/
public Builder withRuntime(@Nullable final Runtime runtime) {
this.bImages.clear();
if (runtime == null) {
this.bRuntimeResources
.withCpu(null)
.withGpu(null)
.withMemoryMb(null)
.withDiskMb(null)
.withNetworkMbps(null);
} else {
final RuntimeResources resources = runtime.getResources();
this.bRuntimeResources
.withCpu(resources.getCpu().orElse(null))
.withGpu(resources.getGpu().orElse(null))
.withMemoryMb(resources.getMemoryMb().orElse(null))
.withDiskMb(resources.getDiskMb().orElse(null))
.withNetworkMbps(resources.getNetworkMbps().orElse(null));
this.bImages.putAll(runtime.getImages());
}

return this;
}

/**
* Build the job request.
*
Expand Down
Loading

0 comments on commit 8647973

Please sign in to comment.