Preliminary addition of ProgressListener API for FTP and SFTP

- Implemented for Download action on both Connections.

src/cucumber/java/io/linuxserver/davos/bdd/ClientStepDefs.java

@@ -22,6 +22,8 @@ import io.linuxserver.davos.transfer.ftp.client.Client;
 import io.linuxserver.davos.transfer.ftp.client.FTPClient;
 import io.linuxserver.davos.transfer.ftp.client.UserCredentials;
 import io.linuxserver.davos.transfer.ftp.connection.Connection;
+import io.linuxserver.davos.transfer.ftp.connection.progress.FTPProgressListener;
+import io.linuxserver.davos.transfer.ftp.connection.progress.ProgressListener;
 
 public class ClientStepDefs {
 
@@ -31,6 +33,7 @@ public class ClientStepDefs {
     private int fakeFtpServerPort;
     private Connection connection;
     private Client client;
+    private ProgressListener progressListener;
 
     @After("@Client")
     public void after() {
@@ -82,7 +85,7 @@ public class ClientStepDefs {
 
     @When("^downloads a file$")
     public void downloads_a_file() throws Throwable {
-        connection.download(new FTPFile("file2.txt", 0, "/tmp", 0, false), TMP);
+        connection.download(new FTPFile("file2.txt", "hello world".getBytes().length, "/tmp/", 0, false), TMP);
     }
 
     @Then("^the file is located in the specified local directory$")
@@ -92,4 +95,34 @@ public class ClientStepDefs {
         assertThat(file.exists()).isTrue();
         file.delete();
     }
+    
+    @When("^initialises a Progress Listener for that connection$")
+    public void initialises_a_Progress_Listener_for_that_connection() throws Throwable {
+        
+        progressListener = new CountingFTPProgressListener();
+        
+        connection.setProgressListener(progressListener);
+    }
+
+    @Then("^the Progress Listener will have its values updated$")
+    public void the_Progress_Listener_will_have_its_values_updated() throws Throwable {
+        
+        assertThat(progressListener.getProgress()).isEqualTo(100);
+        assertThat(((CountingFTPProgressListener) progressListener).getTimesCalled()).isEqualTo(11);
+    }
+    
+    class CountingFTPProgressListener extends FTPProgressListener {
+        
+        int timesCalled;
+        
+        @Override 
+        public void updateBytesWritten(long byteCount) {
+            super.updateBytesWritten(byteCount);
+            timesCalled++;
+        }
+        
+        public int getTimesCalled() {
+            return timesCalled;
+        }
+    }
 }

src/cucumber/resources/Client.feature

@@ -12,4 +12,13 @@ Feature: General client tests
 		Given there is an FTP server running
 		When davos connects to the server
 		And downloads a file
-		Then the file is located in the specified local directory
+		Then the file is located in the specified local directory
+		
+    @Listener
+	Scenario: Download with FTP Progress Listener
+	
+		Given there is an FTP server running
+		When davos connects to the server
+		And initialises a Progress Listener for that connection
+		And downloads a file
+		Then the Progress Listener will have its values updated

src/main/java/io/linuxserver/davos/dto/FTPFileDTO.java

@@ -0,0 +1,11 @@
+package io.linuxserver.davos.dto;
+
+public class FTPFileDTO {
+
+    public String name;
+    public String extension;
+    public String modified;
+    public String path;
+    public long size;
+    public boolean directory;
+}

src/main/java/io/linuxserver/davos/dto/converters/FTPFileDTOConverter.java

@@ -0,0 +1,24 @@
+package io.linuxserver.davos.dto.converters;
+
+import io.linuxserver.davos.dto.FTPFileDTO;
+import io.linuxserver.davos.transfer.ftp.FTPFile;
+
+public class FTPFileDTOConverter implements Converter<FTPFile, FTPFileDTO> {
+
+    @Override
+    public FTPFileDTO convert(FTPFile source) {
+
+        FTPFileDTO dto = new FTPFileDTO();
+
+        dto.directory = source.isDirectory();
+        dto.modified = source.getLastModified().toString("dd MMM yyyy HH:mm:ss");
+        dto.name = source.getName();
+        dto.path = source.getPath();
+        dto.size = source.getSize();
+
+        if (!source.isDirectory())
+            dto.extension = dto.name.substring(dto.name.lastIndexOf('.') + 1, dto.name.length());
+
+        return dto;
+    }
+}

src/main/java/io/linuxserver/davos/transfer/ftp/connection/Connection.java

@@ -3,6 +3,7 @@ package io.linuxserver.davos.transfer.ftp.connection;
 import java.util.List;
 
 import io.linuxserver.davos.transfer.ftp.FTPFile;
+import io.linuxserver.davos.transfer.ftp.connection.progress.ProgressListener;
 import io.linuxserver.davos.transfer.ftp.exception.FTPException;
 
 public interface Connection {
@@ -14,4 +15,6 @@ public interface Connection {
     List<FTPFile> listFiles() throws FTPException;
     
     List<FTPFile> listFiles(String remoteDirectory) throws FTPException;
+    
+    void setProgressListener(ProgressListener progressListener);
 }

src/main/java/io/linuxserver/davos/transfer/ftp/connection/FTPConnection.java

@@ -8,10 +8,13 @@ import java.util.List;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
+import org.apache.commons.io.output.CountingOutputStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import io.linuxserver.davos.transfer.ftp.FTPFile;
+import io.linuxserver.davos.transfer.ftp.connection.progress.FTPProgressListener;
+import io.linuxserver.davos.transfer.ftp.connection.progress.ProgressListener;
 import io.linuxserver.davos.transfer.ftp.exception.DownloadFailedException;
 import io.linuxserver.davos.transfer.ftp.exception.FileListingException;
 import io.linuxserver.davos.util.FileStreamFactory;
@@ -24,6 +27,7 @@ public class FTPConnection implements Connection {
     private org.apache.commons.net.ftp.FTPClient client;
     private FileStreamFactory fileStreamFactory = new FileStreamFactory();
     private FileUtils fileUtils = new FileUtils();
+    private FTPProgressListener progressListener;
 
     public FTPConnection(org.apache.commons.net.ftp.FTPClient client) {
         this.client = client;
@@ -87,13 +91,50 @@ public class FTPConnection implements Connection {
         return files.stream().filter(removeCurrentAndParentDirs()).collect(Collectors.toList());
     }
 
+    @Override
+    public void setProgressListener(ProgressListener progressListener) {
+        this.progressListener = (FTPProgressListener) progressListener;
+    }
+
+    private CountingOutputStream listenOn(OutputStream outputStream) {
+
+        LOGGER.debug("Creating wrapping output stream for progress listener");
+        
+        CountingOutputStream countingStream = new CountingOutputStream(outputStream) {
+
+            @Override
+            protected void beforeWrite(int n) {
+
+                super.beforeWrite(n);
+                progressListener.updateBytesWritten(getByteCount());
+            }
+        };
+
+        return countingStream;
+    }
+
     private void doDownload(FTPFile file, String cleanRemotePath, String cleanLocalPath)
             throws FileNotFoundException, IOException {
 
         LOGGER.info("Downloading {} to {}", cleanRemotePath, cleanLocalPath);
         LOGGER.debug("Creating output stream for file {}", cleanLocalPath + file.getName());
+
         OutputStream outputStream = fileStreamFactory.createOutputStream(cleanLocalPath + file.getName());
-        boolean hasDownloaded = client.retrieveFile(cleanRemotePath, outputStream);
+
+        boolean hasDownloaded;
+
+        if (null != progressListener) {
+         
+            LOGGER.debug("ProgressListener has been set. Initialising...");
+            LOGGER.debug("Total file size is {}", file.getSize());
+            progressListener.reset();
+            progressListener.setTotalSize(file.getSize());
+            
+            hasDownloaded = client.retrieveFile(cleanRemotePath, listenOn(outputStream));
+        }
+        else
+            hasDownloaded = client.retrieveFile(cleanRemotePath, outputStream);
+
         outputStream.close();
 
         if (!hasDownloaded)

src/main/java/io/linuxserver/davos/transfer/ftp/connection/SFTPConnection.java

@@ -14,6 +14,8 @@ import com.jcraft.jsch.ChannelSftp.LsEntry;
 import com.jcraft.jsch.SftpException;
 
 import io.linuxserver.davos.transfer.ftp.FTPFile;
+import io.linuxserver.davos.transfer.ftp.connection.progress.ProgressListener;
+import io.linuxserver.davos.transfer.ftp.connection.progress.SFTPProgressListener;
 import io.linuxserver.davos.transfer.ftp.exception.DownloadFailedException;
 import io.linuxserver.davos.transfer.ftp.exception.FileListingException;
 import io.linuxserver.davos.util.FileUtils;
@@ -24,6 +26,7 @@ public class SFTPConnection implements Connection {
 
     private ChannelSftp channel;
     private FileUtils fileUtils = new FileUtils();
+    private SFTPProgressListener progressListener;
 
     public SFTPConnection(ChannelSftp channel) {
         this.channel = channel;
@@ -50,7 +53,7 @@ public class SFTPConnection implements Connection {
             if (file.isDirectory())
                 downloadDirectoryAndContents(file, cleanLocalPath, path);
             else
-                channel.get(path, cleanLocalPath);
+                doGet(path, cleanLocalPath);
 
         } catch (SftpException e) {
             throw new DownloadFailedException("Unable to download file " + path, e);
@@ -88,6 +91,22 @@ public class SFTPConnection implements Connection {
         }
     }
 
+    @Override
+    public void setProgressListener(ProgressListener progressListener) {
+        this.progressListener = (SFTPProgressListener) progressListener;
+    }
+
+    private void doGet(String fullRemotePath, String fullLocalDownloadPath) throws SftpException {
+
+        if (null != progressListener) {
+
+            LOGGER.debug("Progress listener has been enabled");
+            channel.get(fullRemotePath, fullLocalDownloadPath, progressListener);
+
+        } else
+            channel.get(fullRemotePath, fullLocalDownloadPath);
+    }
+
     private void downloadDirectoryAndContents(FTPFile file, String localDownloadFolder, String path) throws SftpException {
 
         LOGGER.info("Item {} is a directory. Will now check sub-items", file.getName());
@@ -111,7 +130,7 @@ public class SFTPConnection implements Connection {
             else {
 
                 LOGGER.info("Downloading {} to {}", subItemPath, fullLocalDownloadPath);
-                channel.get(subItemPath, fullLocalDownloadPath);
+                doGet(subItemPath, fullLocalDownloadPath);
             }
         }
     }

src/main/java/io/linuxserver/davos/transfer/ftp/connection/progress/FTPProgressListener.java

@@ -0,0 +1,26 @@
+package io.linuxserver.davos.transfer.ftp.connection.progress;
+
+public class FTPProgressListener implements ProgressListener {
+
+    private long totalSize;
+    private long byteCount;
+
+    @Override
+    public double getProgress() {
+        return ((double) byteCount / (double) totalSize) * 100;
+    }
+    
+    public void setTotalSize(long totalSize) {
+        this.totalSize = totalSize;
+    }
+    
+    public void updateBytesWritten(long byteCount) {
+        this.byteCount = byteCount;
+    }
+
+    @Override
+    public void reset() {
+        totalSize = 0;
+        byteCount = 0;
+    }
+}

src/main/java/io/linuxserver/davos/transfer/ftp/connection/progress/ProgressListener.java

@@ -0,0 +1,7 @@
+package io.linuxserver.davos.transfer.ftp.connection.progress;
+
+public interface ProgressListener {
+
+    double getProgress();
+    void reset();
+}

src/main/java/io/linuxserver/davos/transfer/ftp/connection/progress/SFTPProgressListener.java

@@ -0,0 +1,36 @@
+package io.linuxserver.davos.transfer.ftp.connection.progress;
+
+import com.jcraft.jsch.SftpProgressMonitor;
+
+public class SFTPProgressListener implements SftpProgressMonitor, ProgressListener {
+
+    private double progress;
+    private long totalSize;
+
+    @Override
+    public double getProgress() {
+        return (progress / (double) totalSize) * 100;
+    }
+
+    @Override
+    public void init(int op, String src, String dest, long max) {
+        totalSize = max;
+    }
+
+    @Override
+    public boolean count(long count) {
+        progress = count;
+        return true;
+    }
+
+    @Override
+    public void end() {
+        reset();
+    }
+
+    @Override
+    public void reset() {
+        progress = 0;
+        totalSize = 0;
+    }
+}

src/main/java/io/linuxserver/davos/web/ScheduleAPIController.java

@@ -25,10 +25,10 @@ import io.linuxserver.davos.persistence.model.ScheduleConfigurationModel;
 import io.linuxserver.davos.schedule.ScheduleExecutor;
 
 @RestController
-@RequestMapping("/api/v1")
-public class RestAPIController {
+@RequestMapping("/api/v1/schedule")
+public class ScheduleAPIController {
 
-    private static final Logger LOGGER = LoggerFactory.getLogger(RestAPIController.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(ScheduleAPIController.class);
 
     @Resource
     private ScheduleConfigurationDAO scheduleConfigurationDAO;
@@ -36,12 +36,12 @@ public class RestAPIController {
     @Resource
     private ScheduleExecutor scheduleExecutor;
 
-    @RequestMapping(value = "/schedule/{id}")
+    @RequestMapping(value = "/{id}")
     public ScheduleConfigurationDTO getScheduleConfig(@PathVariable("id") Long id) {
         return toDTO(scheduleConfigurationDAO.getConfig(id));
     }
 
-    @RequestMapping(value = "/schedule/{id}/stop")
+    @RequestMapping(value = "/{id}/stop")
     public ScheduleProcessResponse stopScheduleConfig(@PathVariable("id") Long id) {
 
         ScheduleProcessResponse scheduleProcessResponse = new ScheduleProcessResponse();
@@ -58,7 +58,7 @@ public class RestAPIController {
         return scheduleProcessResponse;
     }
 
-    @RequestMapping(value = "/schedule/{id}/start")
+    @RequestMapping(value = "/{id}/start")
     public ScheduleProcessResponse startScheduleConfig(@PathVariable("id") Long id) {
 
         ScheduleProcessResponse scheduleProcessResponse = new ScheduleProcessResponse();
@@ -75,7 +75,7 @@ public class RestAPIController {
         return scheduleProcessResponse;
     }
 
-    @RequestMapping(value = "/schedule", method = RequestMethod.POST)
+    @RequestMapping(method = RequestMethod.POST)
     public ScheduleConfigurationDTO createScheduleConfig(@RequestBody ScheduleConfigurationDTO dto) {
 
         ScheduleConfigurationModel model = new ScheduleConfigurationModelConverter().convert(dto);

src/test/java/io/linuxserver/davos/dto/converters/FTPFileDTOConverterTest.java

@@ -0,0 +1,37 @@
+package io.linuxserver.davos.dto.converters;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.joda.time.DateTime;
+import org.junit.Test;
+
+import io.linuxserver.davos.dto.FTPFileDTO;
+import io.linuxserver.davos.transfer.ftp.FTPFile;
+
+public class FTPFileDTOConverterTest {
+
+    @Test
+    public void shouldConvertFile() {
+        
+        FTPFile file = new FTPFile("name.txt", 12345l, "/some/path", new DateTime(2016, 3, 12, 14, 34, 15).getMillis(), false);
+        
+        FTPFileDTO dto = new FTPFileDTOConverter().convert(file);
+        
+        assertThat(dto.name).isEqualTo("name.txt");
+        assertThat(dto.size).isEqualTo(12345l);
+        assertThat(dto.path).isEqualTo("/some/path");
+        assertThat(dto.modified).isEqualTo("12 Mar 2016 14:34:15");
+        assertThat(dto.extension).isEqualTo("txt");
+        assertThat(dto.directory).isEqualTo(false);
+    }
+    
+    @Test
+    public void extensionShouldBeNullIfDirectory() {
+        
+        FTPFile file = new FTPFile("name", 12345l, "/some/path", new DateTime(2016, 3, 12, 14, 34, 15).getMillis(), true);
+        FTPFileDTO dto = new FTPFileDTOConverter().convert(file);
+        
+        assertThat(dto.extension).isNull();
+        assertThat(dto.directory).isEqualTo(true);
+    }
+}

src/test/java/io/linuxserver/davos/transfer/ftp/connection/FTPConnectionTest.java

@@ -5,6 +5,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.is;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -18,6 +19,7 @@ import java.util.Calendar;
 import java.util.Date;
 import java.util.List;
 
+import org.apache.commons.io.output.CountingOutputStream;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -28,6 +30,7 @@ import org.mockito.Mock;
 import org.mockito.Mockito;
 
 import io.linuxserver.davos.transfer.ftp.FTPFile;
+import io.linuxserver.davos.transfer.ftp.connection.progress.FTPProgressListener;
 import io.linuxserver.davos.transfer.ftp.exception.DownloadFailedException;
 import io.linuxserver.davos.transfer.ftp.exception.FileListingException;
 import io.linuxserver.davos.util.FileStreamFactory;
@@ -141,6 +144,18 @@ public class FTPConnectionTest {
 
         verify(mockFileStreamFactory).createOutputStream(LOCAL_DIRECTORY + "/remote.file");
     }
+    
+    @Test
+    public void downloadMethodShouldCreateLocalFileStreamContainingProgressListener() throws IOException {
+
+        FTPFile file = new FTPFile("remote.file", 0l, "path/to", 0, false);
+        
+        ftpConnection.setProgressListener(new FTPProgressListener());
+        ftpConnection.download(file, LOCAL_DIRECTORY);
+
+        verify(mockFileStreamFactory).createOutputStream(LOCAL_DIRECTORY + "/remote.file");
+        verify(mockFtpClient).retrieveFile(eq("path/to/remote.file"), any(CountingOutputStream.class));
+    }
 
     @Test
     public void downloadMethodShouldCallOnFtpClientRetrieveFilesMethodWithRemoteFilename() throws IOException {

src/test/java/io/linuxserver/davos/transfer/ftp/connection/SFTPConnectionTest.java

@@ -6,6 +6,7 @@ import static org.hamcrest.CoreMatchers.is;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.mockito.MockitoAnnotations.initMocks;
@@ -28,6 +29,7 @@ import com.jcraft.jsch.SftpATTRS;
 import com.jcraft.jsch.SftpException;
 
 import io.linuxserver.davos.transfer.ftp.FTPFile;
+import io.linuxserver.davos.transfer.ftp.connection.progress.SFTPProgressListener;
 import io.linuxserver.davos.transfer.ftp.exception.DownloadFailedException;
 import io.linuxserver.davos.transfer.ftp.exception.FileListingException;
 import io.linuxserver.davos.util.FileUtils;
@@ -151,6 +153,19 @@ public class SFTPConnectionTest {
 
         verify(mockChannel).get("path/name", "some/directory/");
     }
+    
+    @Test
+    public void downloadMethodShouldCallChannelGetMethodWithListenerIfSet() throws SftpException {
+
+        FTPFile file = new FTPFile("name", 0, "path", 0, false);
+        SFTPProgressListener progressListener = new SFTPProgressListener();
+
+        sftpConnection.setProgressListener(progressListener);
+        sftpConnection.download(file, "some/directory");
+
+        verify(mockChannel).get("path/name", "some/directory/", progressListener);
+        verify(mockChannel, never()).get("path/name", "some/directory/");
+    }
 
     @Test
     public void downloadMethodShouldThrowDownloadFailedExceptionWhenChannelThrowsSftpConnection() throws SftpException {

src/test/java/io/linuxserver/davos/transfer/ftp/connection/progress/SFTPProgressListenerTest.java

@@ -0,0 +1,27 @@
+package io.linuxserver.davos.transfer.ftp.connection.progress;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.Test;
+
+import io.linuxserver.davos.transfer.ftp.connection.progress.SFTPProgressListener;
+
+public class SFTPProgressListenerTest {
+
+    @Test
+    public void shouldReturnCorrectProgress() {
+        
+        SFTPProgressListener listener = new SFTPProgressListener();
+        
+        listener.init(0, "", "", 500);
+
+        listener.count(100);
+        assertThat(listener.getProgress()).isEqualTo(20);
+        
+        listener.count(250);
+        assertThat(listener.getProgress()).isEqualTo(50);
+        
+        listener.count(500);
+        assertThat(listener.getProgress()).isEqualTo(100);
+    }
+}