[BACKLOG-26517] Adding Password to ExcelInput dialog (#6036)

* [BACKLOG-26517] Adding Password to ExcelInput dialog

* [BACKLOG-26517] Cleaning up styles and headers (no logic changes)

engine/src/main/java/org/pentaho/di/trans/steps/excelinput/ExcelInput.java

@@ -549,7 +549,7 @@ public Object[] getRowFromWorkbooks() {
             + data.filenr + " : " + data.filename ) );
         }
 
-        data.workbook = WorkbookFactory.getWorkbook( meta.getSpreadSheetType(), data.filename, meta.getEncoding() );
+        data.workbook = WorkbookFactory.getWorkbook( meta.getSpreadSheetType(), data.filename, meta.getEncoding(), meta.getPassword() );
 
         data.errorHandler.handleFile( data.file );
         // Start at the first sheet again...

engine/src/main/java/org/pentaho/di/trans/steps/excelinput/ExcelInputMeta.java

@@ -2,7 +2,7 @@
  *
  * Pentaho Data Integration
  *
- * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com
+ * Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com
  *
  *******************************************************************************
  *
@@ -30,6 +30,7 @@
 import org.pentaho.di.core.CheckResult;
 import org.pentaho.di.core.CheckResultInterface;
 import org.pentaho.di.core.Const;
+import org.pentaho.di.core.encryption.Encr;
 import org.pentaho.di.core.injection.AfterInjection;
 import org.pentaho.di.core.util.StringUtil;
 import org.pentaho.di.core.util.Utils;
@@ -284,6 +285,8 @@ public class ExcelInputMeta extends BaseStepMeta implements StepMetaInterface {
   @Injection( name = "SPREADSHEET_TYPE" )
   private SpreadSheetType spreadSheetType;
 
+  private String password;
+
   public ExcelInputMeta() {
     super(); // allocate BaseStepMeta
   }
@@ -302,6 +305,14 @@ public void setShortFileNameField( String field ) {
     shortFileFieldName = field;
   }
 
+  public String getPassword() {
+    return password;
+  }
+
+  public void setPassword( String password ) {
+    this.password = password;
+  }
+
   /**
    * @return Returns the pathFieldName.
    */
@@ -787,6 +798,8 @@ private void readData( Node stepnode ) throws KettleXMLException {
       extensionFieldName = XMLHandler.getTagValue( stepnode, "extensionFieldName" );
       sizeFieldName = XMLHandler.getTagValue( stepnode, "sizeFieldName" );
 
+      password = Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( stepnode, "password" ) );
+
       try {
         spreadSheetType = SpreadSheetType.valueOf( XMLHandler.getTagValue( stepnode, "spreadsheet_type" ) );
       } catch ( Exception e ) {
@@ -1104,6 +1117,9 @@ public String getXML() {
     retval.append( "    " ).append( XMLHandler.addTagValue( "spreadsheet_type",
       ( spreadSheetType != null ? spreadSheetType.toString() : StringUtil.EMPTY_STRING ) ) );
 
+    retval.append( "    " )
+      .append( XMLHandler.addTagValue( "password", Encr.encryptPasswordIfNotUsingVariables( password ) ) );
+
     return retval.toString();
   }
 
@@ -1195,6 +1211,8 @@ public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, Lis
       extensionFieldName = rep.getStepAttributeString( id_step, "extensionFieldName" );
       sizeFieldName = rep.getStepAttributeString( id_step, "sizeFieldName" );
 
+      password = Encr.decryptPasswordOptionallyEncrypted( rep.getStepAttributeString( id_step, "password" ) );
+
       try {
         spreadSheetType = SpreadSheetType.valueOf( rep.getStepAttributeString( id_step, "spreadsheet_type" ) );
       } catch ( Exception e ) {
@@ -1282,6 +1300,10 @@ public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transform
 
       rep.saveStepAttribute( id_transformation, id_step, "spreadsheet_type",
         ( spreadSheetType != null ? spreadSheetType.toString() : StringUtil.EMPTY_STRING ) );
+
+      rep.saveStepAttribute( id_transformation, id_step, "password", Encr
+        .encryptPasswordIfNotUsingVariables( password ) );
+
     } catch ( Exception e ) {
       throw new KettleException( "Unable to save step information to the repository for id_step=" + id_step, e );
     }
@@ -1660,8 +1682,8 @@ public void setSpreadSheetType( SpreadSheetType spreadSheetType ) {
   }
 
   /**
-   * If we use injection we can have different arrays lengths.
-   * We need synchronize them for consistency behavior with UI
+   * If we use injection we can have different arrays lengths. We need synchronize them for consistency behavior with
+   * UI
    */
   @AfterInjection
   public void afterInjectionSynchronization() {

engine/src/main/java/org/pentaho/di/trans/steps/excelinput/WorkbookFactory.java

@@ -2,7 +2,7 @@
  *
  * Pentaho Data Integration
  *
- * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com
+ * Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com
  *
  *******************************************************************************
  *
@@ -34,11 +34,16 @@
 public class WorkbookFactory {
 
   public static KWorkbook getWorkbook( SpreadSheetType type, String filename, String encoding ) throws KettleException {
+    return getWorkbook( type, filename, encoding, null );
+  }
+
+  public static KWorkbook getWorkbook( SpreadSheetType type, String filename, String encoding, String password )
+    throws KettleException {
     switch ( type ) {
       case JXL:
         return new XLSWorkbook( filename, encoding );
       case POI:
-        return new PoiWorkbook( filename, encoding ); // encoding is not used, perhaps detected automatically?
+        return new PoiWorkbook( filename, encoding, password ); // encoding is not used, perhaps detected automatically?
       case SAX_POI:
         return new StaxPoiWorkbook( filename, encoding );
       case ODS:
@@ -49,7 +54,9 @@ public static KWorkbook getWorkbook( SpreadSheetType type, String filename, Stri
 
   }
 
-  public static KWorkbook getWorkbook( SpreadSheetType type, InputStream inputStream, String encoding ) throws KettleException {
+  // Not Dead Code:  Used by pdi-google-docs-plugin
+  public static KWorkbook getWorkbook( SpreadSheetType type, InputStream inputStream, String encoding )
+    throws KettleException {
     switch ( type ) {
       case JXL:
         return new XLSWorkbook( inputStream, encoding );

engine/src/main/java/org/pentaho/di/trans/steps/excelinput/poi/PoiWorkbook.java

@@ -2,7 +2,7 @@
  *
  * Pentaho Data Integration
  *
- * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com
+ * Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com
  *
  *******************************************************************************
  *
@@ -28,6 +28,7 @@
 
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.provider.local.LocalFile;
+import org.apache.poi.EncryptedDocumentException;
 import org.apache.poi.openxml4j.opc.OPCPackage;
 import org.apache.poi.poifs.filesystem.NPOIFSFileSystem;
 import org.apache.poi.ss.usermodel.Sheet;
@@ -52,6 +53,10 @@ public class PoiWorkbook implements KWorkbook {
   private OPCPackage opcpkg;
 
   public PoiWorkbook( String filename, String encoding ) throws KettleException {
+    this( filename, encoding, null );
+  }
+
+  public PoiWorkbook( String filename, String encoding, String password ) throws KettleException {
     this.filename = filename;
     this.encoding = encoding;
     this.log = KettleLogStore.getLogChannelInterfaceFactory().create( this );
@@ -70,13 +75,16 @@ public PoiWorkbook( String filename, String encoding ) throws KettleException {
             opcpkg = OPCPackage.open( excelFile );
             workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create( opcpkg );
           } catch ( Exception ex ) {
-            workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create( excelFile );
+            workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create( excelFile, password );
           }
         }
       } else {
         internalIS = KettleVFS.getInputStream( filename );
-        workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create( internalIS );
+        workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create( internalIS, password );
       }
+    } catch ( EncryptedDocumentException e ) {
+      log.logError( "Unable to open spreadsheet.  If the spreadsheet is password protected please double check the password is correct." );
+      throw new KettleException( e.getLocalizedMessage() );
     } catch ( Exception e ) {
       throw new KettleException( e );
     }

engine/src/main/resources/org/pentaho/di/trans/steps/excelinput/messages/messages_en_US.properties

@@ -167,3 +167,5 @@ ExcelInput.Injection.SHEET_NAME=Sheet name
 ExcelInput.Injection.SHEET_START_ROW=Sheet start row
 ExcelInput.Injection.SHEET_START_COL=Sheet start col
 ExcelInput.Injection.SPREADSHEET_TYPE=Specify what backend spreadsheet library to use (poi, jxl, etc)
+
+ExeclInputDialog.Password.Label=Password

engine/src/test/java/org/pentaho/di/trans/steps/excelinput/ExcelInputMetaTest.java

@@ -399,7 +399,8 @@ public void testGetXML() throws KettleException {
         + "    <rootUriNameFieldName/>" + SystemUtils.LINE_SEPARATOR
         + "    <extensionFieldName/>" + SystemUtils.LINE_SEPARATOR
         + "    <sizeFieldName/>" + SystemUtils.LINE_SEPARATOR
-        + "    <spreadsheet_type/>" + SystemUtils.LINE_SEPARATOR, meta.getXML() );
+        + "    <spreadsheet_type/>" + SystemUtils.LINE_SEPARATOR
+        + "    <password>Encrypted </password>" + SystemUtils.LINE_SEPARATOR, meta.getXML() );
   }
 
   @Test

integration/src/it/java/org/pentaho/di/trans/steps/excelinput/PoiWorkBookIT.java

@@ -2,7 +2,7 @@
  *
  * Pentaho Data Integration
  *
- * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com
+ * Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com
  *
  *******************************************************************************
  *
@@ -31,7 +31,12 @@
 import java.nio.channels.FileLock;
 import java.util.Date;
 
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.pentaho.di.core.KettleEnvironment;
 import org.pentaho.di.core.exception.KettleException;
 import org.pentaho.di.core.spreadsheet.KCell;
 import org.pentaho.di.core.spreadsheet.KCellType;
@@ -41,26 +46,57 @@
 
 public class PoiWorkBookIT {
 
+  private String sampleFile = "src/it/resources/sample-file.xlsx";
+  private String sampleFileProtected = "src/it/resources/sample-file-protected.xlsx";
+  private String password = "password";
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    KettleEnvironment.init();
+  }
+
   @Test
   public void testReadData() throws KettleException {
-    readData();
+    readData( sampleFile, null );
+  }
+
+  @Test
+  public void testReadDataEmptyPassword() throws KettleException {
+    readData( sampleFile, "" );
+  }
+
+  @Test
+  public void testReadDataProtected() throws KettleException {
+    readData( sampleFileProtected, password );
+  }
+
+  @Test
+  public void testReadDataProtectedEmptyPassword() throws KettleException {
+    KettleException actualException = null;
+    try {
+      readData( sampleFileProtected, "asdf" );
+    } catch ( KettleException e ) {
+      actualException = e;
+    }
+    assertNotNull( actualException );
+    assertEquals( "\nPassword incorrect\n", actualException.getMessage() );
   }
 
   @Test
   public void testFileDoesNotChange() throws KettleException, IOException {
-    File fileBeforeRead = new File( "src/it/resources/sample-file.xlsx" );
-    readData();
-    File fileAfterRead = new File( "src/it/resources/sample-file.xlsx" );
-    assertTrue( FileUtils.contentEquals(fileBeforeRead, fileAfterRead ) );
+    File fileBeforeRead = new File( sampleFile );
+    readData( sampleFile, null );
+    File fileAfterRead = new File( sampleFile );
+    assertTrue( FileUtils.contentEquals( fileBeforeRead, fileAfterRead ) );
   }
 
   @Test
   public void testResourceFree() throws Exception {
     FileLock lock = null;
     RandomAccessFile randomAccessFile = null;
     try {
-      readData();
-      File fileAfterRead = new File( "src/it/resources/sample-file.xlsx" );
+      readData( sampleFile, null );
+      File fileAfterRead = new File( sampleFile );
       randomAccessFile = new RandomAccessFile( fileAfterRead, "rw" );
       FileChannel fileChannel = randomAccessFile.getChannel();
       lock = fileChannel.tryLock();
@@ -76,8 +112,8 @@ public void testResourceFree() throws Exception {
     }
   }
 
-  private void readData() throws KettleException {
-    KWorkbook workbook = WorkbookFactory.getWorkbook( SpreadSheetType.POI, "src/it/resources/sample-file.xlsx", null );
+  private void readData( String file, String password ) throws KettleException {
+    KWorkbook workbook = WorkbookFactory.getWorkbook( SpreadSheetType.POI, file, null, password );
     int numberOfSheets = workbook.getNumberOfSheets();
     assertEquals( 3, numberOfSheets );
     KSheet sheet1 = workbook.getSheet( 0 );