HDDS-1041. Support TDE(Transparent Data Encryption) for Ozone.

Contributed by Xiaoyu Yao.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java

@@ -50,18 +50,27 @@ public final class BucketArgs {
    */
   private Map<String, String> metadata;
 
+  /**
+   * Bucket encryption key name.
+   */
+  private String bucketEncryptionKey;
+
   /**
    * Private constructor, constructed via builder.
    * @param versioning Bucket version flag.
    * @param storageType Storage type to be used.
    * @param acls list of ACLs.
+   * @param metadata map of bucket metadata
+   * @param bucketEncryptionKey bucket encryption key name
    */
   private BucketArgs(Boolean versioning, StorageType storageType,
-                     List<OzoneAcl> acls, Map<String, String> metadata) {
+                     List<OzoneAcl> acls, Map<String, String> metadata,
+                     String bucketEncryptionKey) {
     this.acls = acls;
     this.versioning = versioning;
     this.storageType = storageType;
     this.metadata = metadata;
+    this.bucketEncryptionKey = bucketEncryptionKey;
   }
 
   /**
@@ -97,6 +106,14 @@ public Map<String, String> getMetadata() {
     return metadata;
   }
 
+  /**
+   * Returns the bucket encryption key name.
+   * @return bucket encryption key
+   */
+  public String getEncryptionKey() {
+    return bucketEncryptionKey;
+  }
+
   /**
    * Returns new builder class that builds a OmBucketInfo.
    *
@@ -114,6 +131,7 @@ public static class Builder {
     private StorageType storageType;
     private List<OzoneAcl> acls;
     private Map<String, String> metadata;
+    private String bucketEncryptionKey;
 
     public Builder() {
       metadata = new HashMap<>();
@@ -138,12 +156,18 @@ public BucketArgs.Builder addMetadata(String key, String value) {
       this.metadata.put(key, value);
       return this;
     }
+
+    public BucketArgs.Builder setBucketEncryptionKey(String bek) {
+      this.bucketEncryptionKey = bek;
+      return this;
+    }
     /**
      * Constructs the BucketArgs.
      * @return instance of BucketArgs.
      */
     public BucketArgs build() {
-      return new BucketArgs(versioning, storageType, acls, metadata);
+      return new BucketArgs(versioning, storageType, acls, metadata,
+          bucketEncryptionKey);
     }
   }
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java

@@ -94,6 +94,37 @@ public class OzoneBucket extends WithMetadata {
    */
   private long creationTime;
 
+  /**
+   * Bucket Encryption key name if bucket encryption is enabled.
+   */
+  private String encryptionKeyName;
+
+  @SuppressWarnings("parameternumber")
+  public OzoneBucket(Configuration conf, ClientProtocol proxy,
+                     String volumeName, String bucketName,
+                     List<OzoneAcl> acls, StorageType storageType,
+                     Boolean versioning, long creationTime,
+                     Map<String, String> metadata,
+                     String encryptionKeyName) {
+    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
+    this.proxy = proxy;
+    this.volumeName = volumeName;
+    this.name = bucketName;
+    this.acls = acls;
+    this.storageType = storageType;
+    this.versioning = versioning;
+    this.listCacheSize = HddsClientUtils.getListCacheSize(conf);
+    this.creationTime = creationTime;
+    this.defaultReplication = ReplicationFactor.valueOf(conf.getInt(
+        OzoneConfigKeys.OZONE_REPLICATION,
+        OzoneConfigKeys.OZONE_REPLICATION_DEFAULT));
+    this.defaultReplicationType = ReplicationType.valueOf(conf.get(
+        OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+        OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT));
+    this.metadata = metadata;
+    this.encryptionKeyName = encryptionKeyName;
+  }
+
   /**
    * Constructs OzoneBucket instance.
    * @param conf Configuration object.
@@ -201,6 +232,14 @@ public long getCreationTime() {
     return creationTime;
   }
 
+  /**
+   * Return the bucket encryption key name.
+   * @return the bucket encryption key name
+   */
+  public String getEncryptionKeyName() {
+    return encryptionKeyName;
+  }
+
   /**
    * Adds ACLs to the Bucket.
    * @param addAcls ACLs to be added

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java

@@ -47,6 +47,9 @@ public static BucketInfo asBucketInfo(OzoneBucket bucket) {
     bucketInfo.setVersioning(
         OzoneConsts.Versioning.getVersioning(bucket.getVersioning()));
     bucketInfo.setAcls(bucket.getAcls());
+    bucketInfo.setEncryptionKeyName(
+        bucket.getEncryptionKeyName()==null? "N/A" :
+            bucket.getEncryptionKeyName());
     return bucketInfo;
   }
 
@@ -104,6 +107,7 @@ public static KeyInfoDetails asKeyInfoDetails(OzoneKeyDetails key) {
     key.getOzoneKeyLocations().forEach((a) -> keyLocations.add(new KeyLocation(
         a.getContainerID(), a.getLocalID(), a.getLength(), a.getOffset())));
     keyInfo.setKeyLocation(keyLocations);
+    keyInfo.setFileEncryptionInfo(key.getFileEncryptionInfo());
     return keyInfo;
   }
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.ozone.client;
 
+import org.apache.hadoop.fs.FileEncryptionInfo;
 import org.apache.hadoop.hdds.client.ReplicationType;
 
 import java.util.List;
@@ -35,18 +36,22 @@ public class OzoneKeyDetails extends OzoneKey {
 
   private Map<String, String> metadata;
 
+  private FileEncryptionInfo feInfo;
+
   /**
    * Constructs OzoneKeyDetails from OmKeyInfo.
    */
   @SuppressWarnings("parameternumber")
   public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
                          long size, long creationTime, long modificationTime,
                          List<OzoneKeyLocation> ozoneKeyLocations,
-                         ReplicationType type, Map<String, String> metadata) {
+                         ReplicationType type, Map<String, String> metadata,
+                         FileEncryptionInfo feInfo) {
     super(volumeName, bucketName, keyName, size, creationTime,
         modificationTime, type);
     this.ozoneKeyLocations = ozoneKeyLocations;
     this.metadata = metadata;
+    this.feInfo = feInfo;
   }
 
   /**
@@ -60,6 +65,9 @@ public Map<String, String> getMetadata() {
     return metadata;
   }
 
+  public FileEncryptionInfo getFileEncryptionInfo() {
+    return feInfo;
+  }
   /**
    * Set details of key location.
    * @param ozoneKeyLocations - details of key location

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java

@@ -20,6 +20,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.FileEncryptionInfo;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
@@ -81,6 +82,8 @@ public class KeyOutputStream extends OutputStream {
   private final Checksum checksum;
   private List<ByteBuffer> bufferList;
   private OmMultipartCommitUploadPartInfo commitUploadPartInfo;
+  private FileEncryptionInfo feInfo;
+
   /**
    * A constructor for testing purpose only.
    */
@@ -145,6 +148,9 @@ public KeyOutputStream(OpenKeySession handler,
     this.omClient = omClient;
     this.scmClient = scmClient;
     OmKeyInfo info = handler.getKeyInfo();
+    // Retrieve the file encryption key info, null if file is not in
+    // encrypted bucket.
+    this.feInfo = info.getFileEncryptionInfo();
     this.keyArgs = new OmKeyArgs.Builder().setVolumeName(info.getVolumeName())
         .setBucketName(info.getBucketName()).setKeyName(info.getKeyName())
         .setType(type).setFactor(factor).setDataSize(info.getDataSize())
@@ -547,6 +553,10 @@ public OmMultipartCommitUploadPartInfo getCommitUploadPartInfo() {
     return commitUploadPartInfo;
   }
 
+  public FileEncryptionInfo getFileEncryptionInfo() {
+    return feInfo;
+  }
+
   /**
    * Builder class of KeyOutputStream.
    */

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java

@@ -614,7 +614,8 @@ public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
             bucketInfo.getBucketName(), bucketInfo.getAcls(),
             bucketInfo.getStorageType(),
             getBucketVersioningFlag(bucketInfo.getVersioning()), creationTime,
-            new HashMap<>());
+            new HashMap<>(), bucketInfo
+            .getEncryptionKeyName());
       }).collect(Collectors.toList());
     } catch (URISyntaxException e) {
       throw new IOException(e);
@@ -870,7 +871,7 @@ public OzoneKeyDetails getKeyDetails(
           HddsClientUtils.formatDateTime(keyInfo.getModifiedOn()),
           ozoneKeyLocations, ReplicationType.valueOf(
               keyInfo.getType().toString()),
-          new HashMap<>());
+          new HashMap<>(), keyInfo.getFileEncryptionInfo());
       EntityUtils.consume(response);
       return key;
     } catch (URISyntaxException | ParseException e) {