LAB-linuxserver/davos
1
0
Fork 0
mirror of https://github.com/linuxserver/davos.git synced 2026-04-20 00:01:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implemented full recursion in SFTP downloads

Browse Source 
- If given item to download is a folder, davos will now check all
   contents and download. If any sub-items are folders, it will traverse
   that sub-item and repeat the process until complete.
 - This is a depth-first type algorithm.
This commit is contained in:
Josh Stark 2015-10-19 19:56:57 +01:00
parent e332d356a7
commit 21f78c6bea
11 changed files with 128 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/io/linuxserver/davos/schedule/workflow/transfer/TransferStrategyFactory.java
View File

@ -7,7 +7,7 @@ public class TransferStrategyFactory {
public TransferStrategy getStrategy(FileTransferType type, Connection connection) {
if (FileTransferType.FILES_ONLY.equals(type))
if (FileTransferType.FILE.equals(type))
return new FilesOnlyTransferStrategy(connection);
return new FilesAndFoldersTranferStrategy(connection);

2
src/main/java/io/linuxserver/davos/transfer/ftp/FileTransferType.java
View File

@ -1,5 +1,5 @@
package io.linuxserver.davos.transfer.ftp;
public enum FileTransferType {
FILES_ONLY, INCLUDE_FOLDERS;
FILE, RECURSIVE;
}

42
src/main/java/io/linuxserver/davos/transfer/ftp/connection/SFTPConnection.java
View File

@ -3,6 +3,7 @@ package io.linuxserver.davos.transfer.ftp.connection;
import java.util.ArrayList;
import java.util.List;
import java.util.Vector;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.slf4j.Logger;
@ -42,14 +43,14 @@ public class SFTPConnection implements Connection {
public void download(FTPFile file, String localFilePath) {
String path = FileUtils.ensureTrailingSlash(file.getPath()) + file.getName();
String cleanLocalPath = FileUtils.ensureTrailingSlash(localFilePath);
try {
if (file.isDirectory())
downloadDirectoryAndContents(file, localFilePath, path);
downloadDirectoryAndContents(file, cleanLocalPath, path);
else
channel.get(path, FileUtils.ensureTrailingSlash(localFilePath));
channel.get(path, cleanLocalPath);
} catch (SftpException e) {
throw new DownloadFailedException("Unable to download file " + path, e);
@ -87,27 +88,38 @@ public class SFTPConnection implements Connection {
}
}
private void downloadDirectoryAndContents(FTPFile file, String localFilePath, String path) throws SftpException {
private void downloadDirectoryAndContents(FTPFile file, String localDownloadFolder, String path) throws SftpException {
LOGGER.debug("Item is a directory. Employing single-level directory search for files");
List<FTPFile> subItems = listFiles(path);
List<FTPFile> filesOnly = subItems.stream().filter(f -> !f.isDirectory()).collect(Collectors.toList());
LOGGER.debug("Found {} file(s) in directory. Will attempt to download", filesOnly.size());
LOGGER.info("Item {} is a directory. Will now check sub-items", file.getName());
List<FTPFile> subItems = listFiles(path).stream().filter(removeCurrentAndParentDirs()).collect(Collectors.toList());
for (FTPFile subItem : filesOnly) {
String fullLocalDownloadPath = FileUtils.ensureTrailingSlash(localDownloadFolder + file.getName());
LOGGER.debug("Creating new local directory {}", fullLocalDownloadPath);
fileUtils.createLocalDirectory(fullLocalDownloadPath);
for (FTPFile subItem : subItems) {
String subItemPath = FileUtils.ensureTrailingSlash(subItem.getPath()) + subItem.getName();
String localDirectory = FileUtils.ensureTrailingSlash(file.getName());
String fullLocalDownloadPath = FileUtils.ensureTrailingSlash(localFilePath) + localDirectory;
LOGGER.debug("Creating new local directory {}", fullLocalDownloadPath);
fileUtils.createLocalDirectory(fullLocalDownloadPath);
if (subItem.isDirectory()) {
LOGGER.debug("Downloading {} to {}", subItemPath, fullLocalDownloadPath);
channel.get(subItemPath, fullLocalDownloadPath);
String subLocalFilePath = FileUtils.ensureTrailingSlash(fullLocalDownloadPath);
downloadDirectoryAndContents(subItem, subLocalFilePath, FileUtils.ensureTrailingSlash(subItemPath));
}
else {
LOGGER.info("Downloading {} to {}", subItemPath, fullLocalDownloadPath);
channel.get(subItemPath, fullLocalDownloadPath);
}
}
}
private Predicate<? super FTPFile> removeCurrentAndParentDirs() {
return file -> !file.getName().equals(".") && !file.getName().equals("..");
}
private FTPFile toFtpFile(LsEntry lsEntry, String filePath) throws SftpException {
String name = lsEntry.getFilename();

10
src/main/resources/application.properties
View File

@ -1,5 +1,5 @@
#spring.datasource.url=jdbc:h2:file:~/testdb
#spring.datasource.username=sa
#spring.datasource.password=sa
#spring.datasource.driverClassName=org.h2.Driver
#spring.jpa.hibernate.ddl-auto=update
spring.datasource.url=jdbc:h2:file:~/testdb
spring.datasource.username=sa
spring.datasource.password=sa
spring.datasource.driverClassName=org.h2.Driver
spring.jpa.hibernate.ddl-auto=update

2
src/test/java/io/linuxserver/davos/schedule/ScheduleConfigurationFactoryTest.java
View File

@ -29,7 +29,7 @@ public class ScheduleConfigurationFactoryTest {
model.port = 8;
model.remoteFilePath = "thing/";
model.startAutomatically = true;
model.transferType = FileTransferType.FILES_ONLY;
model.transferType = FileTransferType.FILE;
model.username = "username";
model.lastRun = new DateTime(2015, 1, 1, 0, 0).toDate();

14
src/test/java/io/linuxserver/davos/schedule/workflow/ConnectWorkflowStepTest.java
View File

@ -52,7 +52,7 @@ public class ConnectWorkflowStepTest {
public void runStepShouldCreateNewClient() {
ScheduleConfiguration config = new ScheduleConfiguration("", TransferProtocol.SFTP, "", 0, new UserCredentials("", ""),
"", "", FileTransferType.FILES_ONLY);
"", "", FileTransferType.FILE);
workflowStep.runStep(new ScheduleWorkflow(config));
@ -63,7 +63,7 @@ public class ConnectWorkflowStepTest {
public void runStepShouldSetClientIntoWorkflow() {
ScheduleConfiguration config = new ScheduleConfiguration("", TransferProtocol.SFTP, "", 0, new UserCredentials("", ""),
"", "", FileTransferType.FILES_ONLY);
"", "", FileTransferType.FILE);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
workflowStep.runStep(schedule);
@ -75,7 +75,7 @@ public class ConnectWorkflowStepTest {
public void runStepShouldConnectToNewlyCreatedClient() {
ScheduleConfiguration config = new ScheduleConfiguration("", TransferProtocol.SFTP, "", 0, new UserCredentials("", ""),
"", "", FileTransferType.FILES_ONLY);
"", "", FileTransferType.FILE);
workflowStep.runStep(new ScheduleWorkflow(config));
@ -90,7 +90,7 @@ public class ConnectWorkflowStepTest {
UserCredentials credentials = new UserCredentials(hostIP, hostIP);
ScheduleConfiguration config = new ScheduleConfiguration("scheduleName", TransferProtocol.SFTP, hostIP, port, credentials,
"remotePath", "localPath", FileTransferType.FILES_ONLY);
"remotePath", "localPath", FileTransferType.FILE);
workflowStep.runStep(new ScheduleWorkflow(config));
@ -109,7 +109,7 @@ public class ConnectWorkflowStepTest {
when(mockClient.connect()).thenReturn(mockConnection);
ScheduleConfiguration config = new ScheduleConfiguration("", TransferProtocol.SFTP, "", 0, new UserCredentials("", ""),
"", "", FileTransferType.FILES_ONLY);
"", "", FileTransferType.FILE);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
workflowStep.runStep(schedule);
@ -121,7 +121,7 @@ public class ConnectWorkflowStepTest {
public void runStepShouldCallOnNextStepWhenComplete() {
ScheduleConfiguration config = new ScheduleConfiguration("", TransferProtocol.SFTP, "", 0, new UserCredentials("", ""),
"", "", FileTransferType.FILES_ONLY);
"", "", FileTransferType.FILE);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
workflowStep.runStep(schedule);
@ -136,7 +136,7 @@ public class ConnectWorkflowStepTest {
public void ifClientCannotConnectThenDoNotCallNextStep() {
ScheduleConfiguration config = new ScheduleConfiguration("", TransferProtocol.SFTP, "", 0, new UserCredentials("", ""),
"", "", FileTransferType.FILES_ONLY);
"", "", FileTransferType.FILE);
when(mockClient.connect()).thenThrow(new ClientConnectionException());

4
src/test/java/io/linuxserver/davos/schedule/workflow/DisconnectWorkflowStepTest.java
View File

@ -35,7 +35,7 @@ public class DisconnectWorkflowStepTest {
public void runStepShouldCloseTheConnection() {
ScheduleConfiguration config = new ScheduleConfiguration("", TransferProtocol.SFTP, "", 0, new UserCredentials("", ""),
"", "", FileTransferType.FILES_ONLY);
"", "", FileTransferType.FILE);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
schedule.setConnection(mockConnection);
@ -50,7 +50,7 @@ public class DisconnectWorkflowStepTest {
public void ifDisconnectingFailsThenDoNothing() {
ScheduleConfiguration config = new ScheduleConfiguration("", TransferProtocol.SFTP, "", 0, new UserCredentials("", ""),
"", "", FileTransferType.FILES_ONLY);
"", "", FileTransferType.FILE);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
schedule.setConnection(mockConnection);

8
src/test/java/io/linuxserver/davos/schedule/workflow/DownloadFilesWorkflowStepTest.java
View File

@ -73,7 +73,7 @@ public class DownloadFilesWorkflowStepTest {
workflowStep.setFilesToDownload(filesToDownload);
ScheduleConfiguration config = new ScheduleConfiguration("", TransferProtocol.SFTP, "", 0, new UserCredentials("", ""),
"", "local/", FileTransferType.FILES_ONLY);
"", "local/", FileTransferType.FILE);
ArrayList<PostDownloadAction> actions = new ArrayList<PostDownloadAction>();
actions.add(new MoveFileAction("", ""));
@ -84,7 +84,7 @@ public class DownloadFilesWorkflowStepTest {
schedule.setConnection(mockConnection);
workflowStep.runStep(schedule);
verify(mockTransferStrategyFactory).getStrategy(FileTransferType.FILES_ONLY, mockConnection);
verify(mockTransferStrategyFactory).getStrategy(FileTransferType.FILE, mockConnection);
verify(mockTransferStrategy).setPostDownloadActions(actions);
verify(mockTransferStrategy).transferFile(file, "local/");
@ -101,7 +101,7 @@ public class DownloadFilesWorkflowStepTest {
workflowStep.setFilesToDownload(filesToDownload);
ScheduleConfiguration config = new ScheduleConfiguration("", TransferProtocol.SFTP, "", 0, new UserCredentials("", ""),
"", "local/", FileTransferType.FILES_ONLY);
"", "local/", FileTransferType.FILE);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
schedule.setConnection(mockConnection);
@ -123,7 +123,7 @@ public class DownloadFilesWorkflowStepTest {
workflowStep.setFilesToDownload(filesToDownload);
ScheduleConfiguration config = new ScheduleConfiguration("", TransferProtocol.SFTP, "", 0, new UserCredentials("", ""),
"", "local/", FileTransferType.FILES_ONLY);
"", "local/", FileTransferType.FILE);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
schedule.setConnection(mockConnection);

14
src/test/java/io/linuxserver/davos/schedule/workflow/FilterFilesWorkflowStepTest.java
View File

@ -47,7 +47,7 @@ public class FilterFilesWorkflowStepTest {
@Test
public void workflowStepShouldListFilesInTheRemoteDirectory() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILES_ONLY);
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILE);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
schedule.setConnection(mockConnection);
@ -60,7 +60,7 @@ public class FilterFilesWorkflowStepTest {
@Test
public void workflowStepShouldFilterOutAnyFilesThatAreNotInTheGivenConfigList() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILES_ONLY);
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILE);
config.setFilters(Arrays.asList("file1", "file2", "file4"));
config.setLastRun(DateTime.now().minusDays(1));
@ -91,7 +91,7 @@ public class FilterFilesWorkflowStepTest {
@Test
public void workflowStepShouldFilterOutAnyFilesThatAreNotInTheGivenConfigListAndWereModifiedBeforeLastRun() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILES_ONLY);
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILE);
config.setFilters(Arrays.asList("file1", "file2", "file4"));
config.setLastRun(DateTime.now().minusDays(1));
@ -122,7 +122,7 @@ public class FilterFilesWorkflowStepTest {
@Test
public void workflowStepShouldFilterOutAnyFilesThatDoNotMatchTheWildcards() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILES_ONLY);
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILE);
config.setFilters(Arrays.asList("file1?and?Stuff", "file2*something", "file4*", "file5"));
config.setLastRun(DateTime.now().minusDays(1));
@ -153,7 +153,7 @@ public class FilterFilesWorkflowStepTest {
@Test
public void workflowStepShouldCallNextStepRunMethodOnceSettingFilters() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILES_ONLY);
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILE);
config.setFilters(Arrays.asList("file1", "file2", "file4"));
config.setLastRun(DateTime.now().minusDays(1));
@ -181,7 +181,7 @@ public class FilterFilesWorkflowStepTest {
@Test
public void ifFilterListIsInitiallyEmptyThenAssumeThatAllFilesAfterLastRunShouldBeDownloaded() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILES_ONLY);
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILE);
config.setLastRun(DateTime.now().minusDays(1));
ArrayList<FTPFile> files = new ArrayList<FTPFile>();
@ -211,7 +211,7 @@ public class FilterFilesWorkflowStepTest {
@Test
public void ifListingFilesIsUnsuccessfulThenDoNotCallNextStepAndCallBackupStepInstead() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILES_ONLY);
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/", FileTransferType.FILE);
when(mockConnection.listFiles("remote/")).thenThrow(new FileListingException());

4
src/test/java/io/linuxserver/davos/schedule/workflow/transfer/TransferStrategyFactoryTest.java
View File

@ -11,10 +11,10 @@ public class TransferStrategyFactoryTest {
@Test
public void shouldReturnCorrectStrategies() {
assertThat(new TransferStrategyFactory().getStrategy(FileTransferType.FILES_ONLY, null))
assertThat(new TransferStrategyFactory().getStrategy(FileTransferType.FILE, null))
.isInstanceOf(FilesOnlyTransferStrategy.class);
assertThat(new TransferStrategyFactory().getStrategy(FileTransferType.INCLUDE_FOLDERS, null))
assertThat(new TransferStrategyFactory().getStrategy(FileTransferType.RECURSIVE, null))
.isInstanceOf(FilesAndFoldersTranferStrategy.class);
}
}

91
src/test/java/io/linuxserver/davos/transfer/ftp/connection/SFTPConnectionTest.java
View File

@ -17,8 +17,10 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.mockito.InOrder;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.ChannelSftp.LsEntry;
@ -37,15 +39,15 @@ public class SFTPConnectionTest {
@Mock
private FileUtils mockFileUtils;
private ChannelSftp mockChannel;
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Before
public void setUp() throws SftpException {
mockChannel = mock(ChannelSftp.class);
Vector<LsEntry> lsEntries = createEntries();
@ -84,7 +86,7 @@ public class SFTPConnectionTest {
sftpConnection.listFiles();
}
@Test
public void lsEntriesReturnedFromChannelShouldBeParsedIntoFtpFileAndReturnedInList() {
@ -115,32 +117,32 @@ public class SFTPConnectionTest {
assertThat(files.get(1).getLastModified().toString("dd/MM/yyyy HH:mm:ss")).isEqualTo("12/03/2014 19:22:41");
assertThat(files.get(2).getLastModified().toString("dd/MM/yyyy HH:mm:ss")).isEqualTo("08/02/2014 17:09:24");
}
@Test
public void printingWorkingDirectoryShouldCallOnUnderlyingClientMethodToGetCurrentDirectory() throws SftpException {
sftpConnection.currentDirectory();
verify(mockChannel).pwd();
}
@Test
public void printingWorkingDirectoryShouldReturnExactlyWhatTheUnderlyingClientReturns() {
assertThat(sftpConnection.currentDirectory()).isEqualTo("a/directory");
}
@Test
public void ifClientThrowsExceptionWhenTryingToGetWorkingDirectoryThenCatchExceptionAndRethrow() throws SftpException {
public void ifClientThrowsExceptionWhenTryingToGetWorkingDirectoryThenCatchExceptionAndRethrow() throws SftpException {
expectedException.expect(FileListingException.class);
expectedException.expectMessage(is(equalTo("Unable to print the working directory")));
when(mockChannel.pwd()).thenThrow(new SftpException(0, ""));
sftpConnection.currentDirectory();
}
@Test
public void downloadMethodShouldCallChannelGetMethodWithFtpFileNameAndDirectory() throws SftpException {
@ -160,18 +162,67 @@ public class SFTPConnectionTest {
sftpConnection.download(new FTPFile("file.txt", 0, "path/to", 0, false), "some/directory");
}
@Test
public void downloadShouldRecursivelyCheckFileIfFolderThenLsThatAndGetOnlyFiles() throws SftpException {
initRecursiveListings();
FTPFile directory = new FTPFile("folder", 0, "path/to", 0, true);
sftpConnection.download(directory, "some/directory");
verify(mockChannel).ls("path/to/folder/");
verify(mockFileUtils).createLocalDirectory("some/directory/folder/");
verify(mockChannel).get("path/to/folder/File 2", "some/directory/folder/");
verify(mockChannel).ls("path/to/folder/");
verify(mockFileUtils).createLocalDirectory("some/directory/folder/directory1/");
verify(mockChannel).ls("path/to/folder/directory1/");
verify(mockFileUtils).createLocalDirectory("some/directory/folder/directory1/directory2/");
verify(mockChannel).ls("path/to/folder/directory1/directory2/");
InOrder inOrder = Mockito.inOrder(mockChannel);
inOrder.verify(mockChannel).get("path/to/folder/file1.txt", "some/directory/folder/");
inOrder.verify(mockChannel).get("path/to/folder/file2.txt", "some/directory/folder/");
inOrder.verify(mockChannel).get("path/to/folder/directory1/file3.txt", "some/directory/folder/directory1/");
inOrder.verify(mockChannel).get("path/to/folder/directory1/directory2/file5.txt", "some/directory/folder/directory1/directory2/");
inOrder.verify(mockChannel).get("path/to/folder/directory1/directory2/file6.txt", "some/directory/folder/directory1/directory2/");
inOrder.verify(mockChannel).get("path/to/folder/directory1/file4.txt", "some/directory/folder/directory1/");
}
private void initRecursiveListings() throws SftpException {
Vector<LsEntry> entries = new Vector<LsEntry>();
entries.add(createSingleEntry(".", 123l, 1394525265, true));
entries.add(createSingleEntry("..", 123l, 1394525265, true));
entries.add(createSingleEntry("file1.txt", 123l, 1394525265, false));
entries.add(createSingleEntry("file2.txt", 456l, 1394652161, false));
entries.add(createSingleEntry("directory1", 789l, 1391879364, true));
when(mockChannel.ls("path/to/folder/")).thenReturn(entries);
Vector<LsEntry> subEntries = new Vector<LsEntry>();
subEntries.add(createSingleEntry(".", 123l, 1394525265, true));
subEntries.add(createSingleEntry("..", 123l, 1394525265, true));
subEntries.add(createSingleEntry("file3.txt", 789l, 1394525265, false));
subEntries.add(createSingleEntry("directory2", 789l, 1394525265, true));
subEntries.add(createSingleEntry("file4.txt", 789l, 1394525265, false));
when(mockChannel.ls("path/to/folder/directory1/")).thenReturn(subEntries);
Vector<LsEntry> subSubEntries = new Vector<LsEntry>();
subSubEntries.add(createSingleEntry(".", 123l, 1394525265, true));
subSubEntries.add(createSingleEntry("..", 123l, 1394525265, true));
subSubEntries.add(createSingleEntry("file5.txt", 789l, 1394525265, false));
subSubEntries.add(createSingleEntry("file6.txt", 789l, 1394525265, false));
when(mockChannel.ls("path/to/folder/directory1/directory2/")).thenReturn(subSubEntries);
}
private Vector<LsEntry> createEntries() {
Vector<LsEntry> vector = new Vector<LsEntry>();