Merge pull request #59 from cdapio/uncompress

Decompress response body string if it was compressed
cdapio · Feb 10, 2023 · 15d6bcb · 15d6bcb
2 parents f0a4d91 + 31ebcff
commit 15d6bcb
Show file tree

Hide file tree

Showing 14 changed files with 137 additions and 48 deletions.
diff --git a/checkstyle.xml b/checkstyle.xml
@@ -70,9 +70,6 @@ page at http://checkstyle.sourceforge.net/config.html -->
   <!-- All Java AST specific tests live under TreeWalker module. -->
   <module name="TreeWalker">
 
-    <!-- required for SupressionCommentFilter and SuppressWithNearbyCommentFilter -->
-    <module name="FileContentsHolder"/>
-
     <!--
 
     IMPORT CHECKS
@@ -97,12 +94,15 @@ page at http://checkstyle.sourceforge.net/config.html -->
       <!-- This ensures that static imports go to the end. -->
       <property name="option" value="bottom"/>
       <property name="tokens" value="STATIC_IMPORT, IMPORT"/>
+      <property name="separated" value="true"/>
     </module>
 
     <module name="IllegalImport">
       <property name="illegalPkgs" value="junit.framework"/>
     </module>
 
+    <module name="UnusedImports"/>
+
     <!--
 
     METHOD LENGTH CHECKS
@@ -125,14 +125,10 @@ page at http://checkstyle.sourceforge.net/config.html -->
     <!-- Checks for Javadoc comments.                     -->
     <!-- See http://checkstyle.sf.net/config_javadoc.html -->
     <module name="JavadocMethod">
-      <property name="scope" value="protected"/>
+      <property name="accessModifiers" value="public, protected, package"/>
       <property name="severity" value="error"/>
-      <property name="allowMissingJavadoc" value="true"/>
       <property name="allowMissingParamTags" value="true"/>
       <property name="allowMissingReturnTag" value="true"/>
-      <property name="allowMissingThrowsTags" value="true"/>
-      <property name="allowThrowsTagsForSubclasses" value="true"/>
-      <property name="allowUndeclaredRTE" value="true"/>
     </module>
 
     <module name="JavadocType">
@@ -234,26 +230,10 @@ page at http://checkstyle.sourceforge.net/config.html -->
 
     <!--
 
-    LENGTH and CODING CHECKS
+    CODING CHECKS
 
     -->
 
-    <module name="LineLength">
-      <!-- Checks if a line is too long. -->
-      <property name="max" value="120" default="120"/>
-      <property name="severity" value="error"/>
-
-      <!--
-        The default ignore pattern exempts the following elements:
-          - import statements
-          - long URLs inside comments
-      -->
-
-      <property name="ignorePattern"
-          value="${com.puppycrawl.tools.checkstyle.checks.sizes.LineLength.ignorePattern}"
-          default="^(package .*;\s*)|(import .*;\s*)|( *\* *https?://.*)$"/>
-    </module>
-
     <module name="LeftCurly">
       <!-- Checks for placement of the left curly brace ('{'). -->
       <property name="severity" value="error"/>
@@ -303,6 +283,14 @@ page at http://checkstyle.sourceforge.net/config.html -->
     </module>
 
 
+    <!-- Requires overriding equals and hashCode at the same time -->
+    <module name="EqualsHashCode"/>
+
+    <!-- Empty catch block is not allowed -->
+    <module name="EmptyCatchBlock"/>
+
+    <module name="ExplicitInitialization"/>
+
     <!--
 
     MODIFIERS CHECKS
@@ -317,6 +305,14 @@ page at http://checkstyle.sourceforge.net/config.html -->
         -->
     </module>
 
+    <module name="RedundantModifier">
+      <!-- Checks for redundant modifiers in:
+           - interface and annotation definitions,
+           - the final modifier on methods of final classes, and
+           - inner interface declarations that are declared as static.
+        -->
+    </module>
+
 
     <!--
 
@@ -376,17 +372,45 @@ page at http://checkstyle.sourceforge.net/config.html -->
       <property name="severity" value="error"/>
     </module>
 
+    <module name="SuppressionCommentFilter">
+      <property name="offCommentFormat" value="CHECKSTYLE OFF: (.+)" />
+      <property name="onCommentFormat" value="CHECKSTYLE ON" />
+      <property name="checkFormat" value="Javadoc.*"/>
+      <property name="messageFormat" value="$1"/>
+    </module>
+
   </module>
 
-  <module name="SuppressionFilter">
-    <property name="file" value="suppressions.xml"/>
+  <!--
+
+  LENGTH CHECKS
+
+  -->
+
+  <module name="LineLength">
+    <!-- Checks if a line is too long. -->
+    <property name="max" value="120" default="120"/>
+    <property name="severity" value="error"/>
+    <property name="fileExtensions" value="java, scala"/>
+
+    <!--
+      The default ignore pattern exempts the following elements:
+        - import statements
+        - long URLs inside comments
+    -->
+
+    <property name="ignorePattern"
+              value="${com.puppycrawl.tools.checkstyle.checks.sizes.LineLength.ignorePattern}"
+              default="^(package .*;\s*)|(import .*;\s*)|( *\* *https?://.*)$"/>
   </module>
 
-  <module name="SuppressionCommentFilter">
-    <property name="offCommentFormat" value="CHECKSTYLE OFF: (.+)" />
-    <property name="onCommentFormat" value="CHECKSTYLE ON" />
-    <property name="checkFormat" value="Javadoc.*"/>
-    <property name="messageFormat" value="$1"/>
+  <!--
+    Optional suppression filter. It is optional because when running with Maven, it should be the
+     checkstyle plugin who provides it. It is only used when this file is used in IntelliJ.
+    -->
+  <module name="SuppressionFilter">
+    <property name="file" value="suppressions.xml"/>
+    <property name="optional" value="true"/>
   </module>
 
 </module>
diff --git a/common-cli/src/main/java/io/cdap/common/cli/util/HelpFormatter.java b/common-cli/src/main/java/io/cdap/common/cli/util/HelpFormatter.java
@@ -26,5 +26,5 @@
  */
 public interface HelpFormatter {
 
-  public void print(Iterable<Command> commands, PrintStream printStream);
+  void print(Iterable<Command> commands, PrintStream printStream);
 }
diff --git a/common-cli/src/main/java/io/cdap/common/cli/util/Parser.java b/common-cli/src/main/java/io/cdap/common/cli/util/Parser.java
@@ -33,7 +33,7 @@ public class Parser {
   private static final char ARG_WRAPPER = '"';
   private static final char JSON_WRAPPER = '\'';
 
-  private static enum State {
+  private enum State {
     EMPTY, IN_QUOTES, IN_DOUBLE_QUOTES, IN_MANDATORY_ARG, IN_OPTIONAL_PART
   }
 

diff --git a/common-core/src/main/java/io/cdap/common/Bytes.java b/common-core/src/main/java/io/cdap/common/Bytes.java
@@ -279,6 +279,7 @@ public static String toStringBinary(final byte [] b, int off, int len) {
         }
       }
     } catch (UnsupportedEncodingException e) {
+      //no-op
     }
     return result.toString();
   }
@@ -813,7 +814,7 @@ public static int compareTo(byte[] buffer1, int offset1, int length1,
   }
 
   interface Comparer<T> {
-    public abstract int compareTo(T buffer1, int offset1, int length1, T buffer2, int offset2, int length2);
+    int compareTo(T buffer1, int offset1, int length1, T buffer2, int offset2, int length2);
   }
 
   static Comparer<byte[]> lexicographicalComparerJavaImpl() {

diff --git a/common-core/src/main/java/io/cdap/common/DirUtils.java b/common-core/src/main/java/io/cdap/common/DirUtils.java
@@ -35,7 +35,7 @@ public final class DirUtils {
    * Utility classes should have a public constructor or a default constructor
    * hence made it private.
    */
-  private DirUtils(){}
+  private DirUtils() {}
 
   /**
    * Wipes out all the a directory starting from a given directory.

diff --git a/common-core/src/main/java/io/cdap/common/StringUtils.java b/common-core/src/main/java/io/cdap/common/StringUtils.java
@@ -91,7 +91,9 @@ public static String[] getTrimmedStrings(String str) {
    */
 
   public static String arrayToString(String[] strs) {
-    if (strs.length == 0) { return ""; }
+    if (strs.length == 0) {
+      return "";
+    }
     StringBuilder sbuf = new StringBuilder();
     sbuf.append(strs[0]);
     for (int idx = 1; idx < strs.length; idx++) {
@@ -106,7 +108,7 @@ public static String arrayToString(String[] strs) {
    * which can be represented by a 64-bit integer.
    * TraditionalBinaryPrefix symbol are case insensitive.
    */
-  public static enum TraditionalBinaryPrefix {
+  public enum TraditionalBinaryPrefix {
     KILO(1024),
     MEGA(KILO.value << 10),
     GIGA(MEGA.value << 10),

diff --git a/common-http/src/main/java/io/cdap/common/http/HttpResponse.java b/common-http/src/main/java/io/cdap/common/http/HttpResponse.java
@@ -20,9 +20,11 @@
 import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.Multimap;
 import com.google.common.io.ByteStreams;
+import com.google.common.net.HttpHeaders;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
@@ -32,6 +34,8 @@
 import java.nio.charset.Charset;
 import java.util.List;
 import java.util.Map;
+import java.util.zip.DeflaterInputStream;
+import java.util.zip.GZIPInputStream;
 import javax.annotation.Nullable;
 
 /**
@@ -86,12 +90,44 @@ public byte[] getResponseBody() {
     return responseBody;
   }
 
+  /**
+   * Decodes and return the response body based on the content encoding.
+   */
+  public byte[] getUncompressedResponseBody() {
+    String encoding = headers.entries().stream()
+      .filter(e -> HttpHeaders.CONTENT_ENCODING.equalsIgnoreCase(e.getKey()))
+      .map(Map.Entry::getValue)
+      .findFirst()
+      .orElse(null);
+
+    if (encoding == null) {
+      return responseBody;
+    }
+
+    try {
+      if ("gzip".equalsIgnoreCase(encoding)) {
+        try (InputStream is = new GZIPInputStream(new ByteArrayInputStream(responseBody))) {
+          return ByteStreams.toByteArray(is);
+        }
+      }
+      if ("deflate".equalsIgnoreCase(encoding)) {
+        try (InputStream is = new DeflaterInputStream(new ByteArrayInputStream(responseBody))) {
+          return ByteStreams.toByteArray(is);
+        }
+      }
+    } catch (IOException e) {
+      throw new IllegalStateException(String.format("Failed to decompress %s encoded response body", encoding), e);
+    }
+
+    throw new IllegalStateException("Unsupported content encoding " + encoding);
+  }
+
   public String getResponseBodyAsString() {
-    return new String(responseBody, Charsets.UTF_8);
+    return getResponseBodyAsString(Charsets.UTF_8);
   }
 
   public String getResponseBodyAsString(Charset charset) {
-    return new String(responseBody, charset);
+    return new String(getUncompressedResponseBody(), charset);
   }
 
   public Multimap<String, String> getHeaders() {

diff --git a/common-http/src/test/java/io/cdap/common/http/HttpRequestsTestBase.java b/common-http/src/test/java/io/cdap/common/http/HttpRequestsTestBase.java
@@ -30,6 +30,7 @@
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -39,6 +40,7 @@
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
+import java.util.zip.GZIPOutputStream;
 import javax.ws.rs.DELETE;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
@@ -67,6 +69,10 @@ public void testHttpStatus() throws Exception {
     testGet("/fake/fake", only(404), only("Not Found"),
             only("Problem accessing: /fake/fake. Reason: Not Found"), any());
     testGet("/api/testOkWithResponse", only(200), any(), only("Great response"), any());
+    if (!returnResponseStream()) {
+      // We don't call getResponseBody() for streaming responses, so skip this test
+      testGet("/api/testOkWithCompressedResponse", only(200), any(), only("Great response"), any());
+    }
 
     int numConnectionsOpened = getNumConnectionsOpened();
     testGet("/api/testOkWithResponse201", only(201), any(), only("Great response 201"), any());
@@ -244,6 +250,19 @@ public void testOkWithResponse(io.netty.handler.codec.http.HttpRequest request,
       responder.sendString(HttpResponseStatus.OK, "Great response");
     }
 
+    @GET
+    @Path("/testOkWithCompressedResponse")
+    public void testOkWithCompressedResponse(io.netty.handler.codec.http.HttpRequest request, HttpResponder responder)
+      throws IOException {
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+      try (GZIPOutputStream gzos = new GZIPOutputStream(baos)) {
+        gzos.write("Great response".getBytes(StandardCharsets.UTF_8));
+      }
+      responder.sendByteArray(HttpResponseStatus.OK, baos.toByteArray(),
+                              new DefaultHttpHeaders().set("Content-Encoding", "gzip"));
+    }
+
     @GET
     @Path("/testOkWithResponse201")
     public void testOkWithResponse201(io.netty.handler.codec.http.HttpRequest request, HttpResponder responder) {

diff --git a/common-io/src/main/java/io/cdap/common/internal/io/Schema.java b/common-io/src/main/java/io/cdap/common/internal/io/Schema.java
@@ -62,7 +62,7 @@ public enum Type {
 
     private final boolean simpleType;
 
-    private Type(boolean primitive) {
+    Type(boolean primitive) {
       this.simpleType = primitive;
     }
 

diff --git a/common-io/src/main/java/io/cdap/common/internal/io/TypeRepresentation.java b/common-io/src/main/java/io/cdap/common/internal/io/TypeRepresentation.java
@@ -37,7 +37,7 @@ public final class TypeRepresentation implements ParameterizedType {
   private final String rawType;
   private final TypeRepresentation enclosingType;
   private final TypeRepresentation[] parameters;
-  private transient ClassLoader classLoader = null;
+  private transient ClassLoader classLoader;
 
   /**
    * Set the class loader to be used by toType(), getRawType(), etc.

diff --git a/common-io/src/test/java/io/cdap/common/io/ASMDatumCodecTest.java b/common-io/src/test/java/io/cdap/common/io/ASMDatumCodecTest.java
@@ -54,7 +54,7 @@ public class ASMDatumCodecTest {
   /**
    *
    */
-  public static enum TestEnum {
+  public enum TestEnum {
     VALUE1, VALUE2, VALUE3, VALUE4
   }
 
@@ -247,7 +247,7 @@ private static class Record {
     private List<String> list;
     private TestEnum e;
 
-    public Record(int i, String s, List<String> list, TestEnum e) {
+    Record(int i, String s, List<String> list, TestEnum e) {
       this.i = i;
       this.s = s;
       this.list = list;

diff --git a/common-io/src/test/java/io/cdap/common/io/DatumCodecTest.java b/common-io/src/test/java/io/cdap/common/io/DatumCodecTest.java
@@ -278,7 +278,7 @@ public void testReduceProjection() throws IOException, UnsupportedTypeException
   /**
    *
    */
-  public static enum TestEnum {
+  public enum TestEnum {
     VALUE1, VALUE2, VALUE3
   }
 

diff --git a/common-lang/pom.xml b/common-lang/pom.xml
@@ -52,4 +52,4 @@ the License.
     </dependency>
   </dependencies>
 
-</project>
+</project>