Java and FTP Client

Java and FTP Client

The same interface with S3FileOperation. The idea is to load the FTP configuration from properties files, upload and download files from remote FTP. I set up mock FTP server to do integration testing.

pom.xml to add commons-net as my client for FTP, MockFtpServer for mock the testing
<dependency>
      <groupId>commons-net</groupId>
      <artifactId>commons-net</artifactId>
      <version>${commons.net.version}</version>
</dependency>
<commons.net.version>3.6</commons.net.version>
    <dependency>
      <groupId>org.mockftpserver</groupId>
      <artifactId>MockFtpServer</artifactId>
      <version>2.7.1</version>
      <scope>test</scope>
    </dependency>

Here is the same interface for both S3 and FTP
package com.sillycat.transmission;

public interface Transmission
{
  public boolean uploadFile( String bucketName, String key, String fileToUpload );
  public boolean downloadFile( String bucketName, String key, String destFolder );
}

Let’s first talk about the unit test, haha, think in test driving implementation.
This is the mock server for testing, so I put that under the test directory
package com.sillycat.transmission;

import org.mockftpserver.fake.FakeFtpServer;
import org.mockftpserver.fake.UserAccount;
import org.mockftpserver.fake.filesystem.DirectoryEntry;
import org.mockftpserver.fake.filesystem.FileSystem;
import org.mockftpserver.fake.filesystem.UnixFakeFileSystem;

public class FtpServerMock
{

  private FakeFtpServer fakeFtpServer;

  public FtpServerMock( int port, String userName, String password, String homeDir )
  {
    fakeFtpServer = new FakeFtpServer();
    fakeFtpServer.setServerControlPort( port );
    fakeFtpServer.addUserAccount( new UserAccount( userName, password, homeDir ) );
  }

  public void start()
  {
    FileSystem fileSystem = new UnixFakeFileSystem();
    fileSystem.add( new DirectoryEntry( "/home" ) );
    fakeFtpServer.setFileSystem( fileSystem );
    fakeFtpServer.start();
  }

  public void stop()
  {
    fakeFtpServer.stop();
  }
}

This is the Integration Test class
package com.sillycat.transmission;

import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

/**
* how to run: mvn -Dtest=TransmissionFtpJUnitIntegrationTest test
*
* @author carl
*
*/
public class TransmissionFtpJUnitIntegrationTest
{

  private Transmission transmission;

  private FtpServerMock serverMock;

  @Before
  public void setUp() throws Exception
  {
    transmission = new TransmissionFtpImpl();
    serverMock = new FtpServerMock( 2121, "carl", "supercarl", "/home" );
    serverMock.start();
  }

  @After
  public void tearDown()
  {
    serverMock.stop();
  }

  @Test
  public void ftpFile()
  {
    String localFilePath = this.getClass().getResource( "mock.txt" ).getPath();
    boolean result = transmission.uploadFile( "mockFtp", "mock2.txt", localFilePath );
    Assert.assertTrue( result );
    localFilePath = localFilePath.substring( 0, localFilePath.indexOf( "mock.txt" ) );
    result = transmission.downloadFile( "mockFtp", "mock2.txt", localFilePath );
    Assert.assertTrue( result );
  }

}

For easier to load the properties from files, I create a java POJO, FtpOption.java
package com.sillycat.transmission;

public class FtpOption
{
  private String host;
  private int port;
  private String user;
  private String password;
  private String root;
  ..snip..
}

The properties format will be similar to this file mockFtp.properties
ftp.host=localhost
ftp.user=carl
ftp.password=supercarl
ftp.root=/home/
ftp.port=2121

This is the TransmissionFtpImpl.java class with will load the properties, find the right ftpName
package com.sillycat.transmission;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.SocketException;
import java.util.Properties;

import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TransmissionFtpImpl implements Transmission
{

  protected final Logger logger = LoggerFactory.getLogger( this.getClass() );

  @Override
  public boolean uploadFile( String ftpName, String path, String fileToUpload )
  {
    boolean result = false;
    FtpOption ftpOption = loadFtpOption( ftpName );
    InputStream input = null;
    FTPClient ftpClient = null;
    try
    {
      ftpClient = loginFtp( ftpOption );
      input = new FileInputStream( fileToUpload );
      ftpClient.storeFile( ftpOption.getRoot() + path, input );
      input.close();
      result = true;
    }
    catch ( IOException e )
    {
      logger.error( "IOException:", e );
    }
    finally
    {
      clearFtpResource( ftpClient );
    }
    return result;
  }

  @Override
  public boolean downloadFile( String ftpName, String path, String destFolder )
  {
    boolean result = false;
    FtpOption ftpOption = loadFtpOption( ftpName );
    FTPClient ftpClient = null;
    try
    {
      ftpClient = loginFtp( ftpOption );
      logger.info( "FTP server " + ftpName + " is " + ftpClient.getSystemType() );
      ftpClient.changeWorkingDirectory( ftpOption.getRoot() );
      FTPFile[] ftpFiles = ftpClient.listFiles();
      if ( ftpFiles != null && ftpFiles.length > 0 )
      {
        for ( FTPFile file : ftpFiles )
        {
          if ( !file.isFile() )
          {
            continue;
          }
          OutputStream output;
          output = new FileOutputStream( destFolder + File.separator + file.getName() );
          //get the file from the remote system
          ftpClient.retrieveFile( file.getName(), output );
          //close output stream
          output.close();
        }
        result = true;
      }
    }
    catch ( IOException e )
    {
      logger.error( "IOException:", e );
    }
    finally
    {
      clearFtpResource( ftpClient );
    }
    return result;
  }

  private FtpOption loadFtpOption( String ftpName )
  {
    FtpOption ftpOption = new FtpOption();
    Properties ftpProperties = new Properties();
    try
    {
      ftpProperties.load( this.getClass().getResourceAsStream( ftpName + ".properties" ) );
    }
    catch ( IOException e )
    {
      logger.error( "IOException:", e );
    }
    String host = ftpProperties.getProperty( "ftp.host" ).trim();
    String user = ftpProperties.getProperty( "ftp.user" ).trim();
    String password = ftpProperties.getProperty( "ftp.password" ).trim();
    String root = ftpProperties.getProperty( "ftp.root" ).trim();
    int port = Integer.valueOf( ftpProperties.getProperty( "ftp.port" ).trim() );

    ftpOption.setHost( host );
    ftpOption.setUser( user );
    ftpOption.setPassword( password );
    ftpOption.setRoot( root );
    ftpOption.setPort( port );
    return ftpOption;
  }

  private FTPClient loginFtp( FtpOption ftpOption ) throws SocketException, IOException
  {
    FTPClient ftpClient = new FTPClient();
    //init connection
    ftpClient.connect( ftpOption.getHost(), ftpOption.getPort() );
    if ( !ftpClient.login( ftpOption.getUser(), ftpOption.getPassword() ) )
    {
      logger.warn( "FTP logging failure with ftpOption:" + ftpOption );
      ftpClient.logout();
    }
    int reply = ftpClient.getReplyCode();
    if ( !FTPReply.isPositiveCompletion( reply ) )
    {
      logger.warn( "FTP did not get positive replay code:" + reply + " ftpOption:" + ftpOption );
      ftpClient.disconnect();
    }
    //enter passive mode
    ftpClient.enterLocalPassiveMode();
    ftpClient.changeWorkingDirectory( ftpOption.getRoot() );
    return ftpClient;
  }

  private void clearFtpResource( FTPClient ftpClient )
  {
    try
    {
      ftpClient.logout();
      ftpClient.disconnect();
    }
    catch ( IOException e )
    {
      logger.error( "IOException:", e );
    }

  }

}

Instead of read the properties files again and again, I may need to add local cache for that as well.
package com.sillycat.cache.guava;

import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.sillycat.cache.CacheService;

public class CacheServiceGuavaImpl<T> implements CacheService<T>
{

  protected final Logger logger = LoggerFactory.getLogger( this.getClass() );

  Cache<String, T> localCache;

  public CacheServiceGuavaImpl()
  {
    localCache = CacheBuilder.newBuilder().expireAfterWrite( 24, TimeUnit.HOURS ).build();
  }

  @Override
  public T load( String key, Callable<T> callable )
  {
    try
    {
      return localCache.get( key, callable );
    }
    catch ( ExecutionException e )
    {
      logger.error( "cache failure:" + e );
    }
    return null;
  }

}

Unit test for that CacheService
package com.sillycat.cache;

import java.util.concurrent.Callable;

import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

import com.sillycat.cache.guava.CacheServiceGuavaImpl;

/**
* how to run: mvn -Dtest=CacheServiceGuavaJUnitTest test
*
* @author carl
*
*/
public class CacheServiceGuavaJUnitTest
{

  private CacheService<String> cacheService;

  @Before
  public void setUp() throws Exception
  {
    cacheService = new CacheServiceGuavaImpl<String>();
  }

  @Test
  public void cache()
  {
    Assert.assertEquals( cacheService.load( "key1", new Callable<String>()
    {
      @Override
      public String call() throws Exception
      {
        return "value1";
      }
    } ), "value1" );
    for ( int i = 0; i < 10;i++ )
    {
      Assert.assertEquals( cacheService.load( "key1", new Callable<String>()
      {
        @Override
        public String call() throws Exception
        {
          return "value1";
        }
      } ), "value1" );
    }
  }

}



References:
http://www.mysamplecode.com/2012/03/apache-commons-ftpclient-java-example_16.html
https://commons.apache.org/proper/commons-net/examples/ftp/FTPClientExample.java
https://commons.apache.org/proper/commons-net/apidocs/org/apache/commons/net/ftp/FTPClient.html
http://www.mysamplecode.com/2012/03/apache-commons-ftpclient-java-example.html
http://www.baeldung.com/properties-with-spring#xml

Mock ftp Server
http://mockftpserver.sourceforge.net/fakeftpserver-getting-started.html
https://itsiastic.wordpress.com/2012/11/08/how-to-create-a-java-ftp-server-mock/