LAB-linuxserver/davos
1
0
Fork 0
mirror of https://github.com/linuxserver/davos.git synced 2026-02-11 18:58:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implemented FilterWorkflowStep

Browse Source
This commit is contained in:
Josh Stark 2015-10-17 16:30:05 +01:00
parent 406da69330
commit b997da42db
16 changed files with 487 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/io/linuxserver/davos/schedule/ScheduleConfiguration.java
View File

@ -3,6 +3,8 @@ package io.linuxserver.davos.schedule;
import java.util.ArrayList;
import java.util.List;
import org.joda.time.DateTime;
import io.linuxserver.davos.schedule.workflow.actions.PostDownloadAction;
import io.linuxserver.davos.transfer.ftp.TransferProtocol;
import io.linuxserver.davos.transfer.ftp.client.UserCredentials;
@ -18,6 +20,7 @@ public class ScheduleConfiguration {
private String scheduleName;
private List<String> filters = new ArrayList<String>();
private List<PostDownloadAction> actions = new ArrayList<PostDownloadAction>();
private DateTime lastRun = DateTime.now();
public ScheduleConfiguration(final String scheduleName, final TransferProtocol protocol, final String hostname,
final int port, final UserCredentials credentials, final String remoteFilePath, final String localFilePath) {
@ -27,6 +30,8 @@ public class ScheduleConfiguration {
this.hostname = hostname;
this.port = port;
this.credentials = credentials;
this.localFilePath = localFilePath;
this.remoteFilePath = remoteFilePath;
}
public TransferProtocol getConnectionType() {
@ -72,4 +77,12 @@ public class ScheduleConfiguration {
public void setActions(List<PostDownloadAction> actions) {
this.actions = actions;
}
public DateTime getLastRun() {
return lastRun;
}
public void setLastRun(DateTime lastRun) {
this.lastRun = lastRun;
}
}

4
src/main/java/io/linuxserver/davos/schedule/workflow/ConnectWorkflowStep.java
View File

@ -37,9 +37,9 @@ public class ConnectWorkflowStep extends WorkflowStep {
} catch (FTPException e) {
LOGGER.warn("Unable to create connection to {} on port {}. Falling back. Will try again next time.",
LOGGER.error("Unable to create connection to {} on port {}. Falling back. Will try again next time.",
schedule.getConfig().getHostName(), schedule.getConfig().getPort());
LOGGER.warn("Error was: {}", e.getMessage());
LOGGER.error("Error was: {}", e.getMessage());
LOGGER.debug("Stacktrace", e);
}
}

11
src/main/java/io/linuxserver/davos/schedule/workflow/DownloadFilesWorkflowStep.java
View File

@ -1,7 +1,14 @@
package io.linuxserver.davos.schedule.workflow;
import java.util.ArrayList;
import java.util.List;
import io.linuxserver.davos.transfer.ftp.FTPFile;
public class DownloadFilesWorkflowStep extends WorkflowStep {
private List<FTPFile> filesToDownload = new ArrayList<FTPFile>();
public DownloadFilesWorkflowStep() {
this.nextStep = new DisconnectWorkflowStep();
}
@ -9,4 +16,8 @@ public class DownloadFilesWorkflowStep extends WorkflowStep {
@Override
public void runStep(ScheduleWorkflow schedule) {
}
public void setFilesToDownload(List<FTPFile> filesToDownload) {
this.filesToDownload = filesToDownload;
}
}

69
src/main/java/io/linuxserver/davos/schedule/workflow/FilterFilesWorkflowStep.java
View File

@ -1,12 +1,79 @@
package io.linuxserver.davos.schedule.workflow;
import static java.util.stream.Collectors.toList;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Predicate;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.linuxserver.davos.transfer.ftp.FTPFile;
import io.linuxserver.davos.transfer.ftp.exception.FTPException;
import io.linuxserver.davos.util.PatternBuilder;
public class FilterFilesWorkflowStep extends WorkflowStep {
private static final Logger LOGGER = LoggerFactory.getLogger(FilterFilesWorkflowStep.class);
public FilterFilesWorkflowStep() {
this.nextStep = new DownloadFilesWorkflowStep();
}
@Override
public void runStep(ScheduleWorkflow schedule) {
try {
DateTime lastRun = schedule.getConfig().getLastRun();
List<String> filters = schedule.getConfig().getFilters();
List<FTPFile> allFiles = schedule.getConnection().listFiles(schedule.getConfig().getRemoteFilePath());
List<FTPFile> filesToFilter = allFiles.stream().filter(after(lastRun)).collect(toList());
List<FTPFile> filteredFiles = new ArrayList<FTPFile>();
if (filters.isEmpty()) {
LOGGER.debug("Filter list was empty. Adding all found files to list");
((DownloadFilesWorkflowStep) nextStep).setFilesToDownload(filesToFilter);
} else {
for (FTPFile file : filesToFilter)
filterFilesByName(filters, filteredFiles, file);
((DownloadFilesWorkflowStep) nextStep).setFilesToDownload(filteredFiles);
}
LOGGER.info("Successfully filtered files. Moving onto next step");
nextStep.runStep(schedule);
} catch (FTPException e) {
LOGGER.error("Unable to filter files. Error message was: {}", e.getMessage());
LOGGER.debug("Stacktrace", e);
}
}
private Predicate<? super FTPFile> after(DateTime lastRun) {
return f -> f.getLastModified().isAfter(lastRun);
}
private void filterFilesByName(List<String> filters, List<FTPFile> filteredFiles, FTPFile file) {
for (String filter : filters) {
String expression = PatternBuilder.buildFromFilterString(filter);
if (file.getName().toLowerCase().matches(expression.toLowerCase())) {
LOGGER.debug("Matched {} to {}. Adding to final filter list.", file.getName().toLowerCase(),
expression.toLowerCase());
filteredFiles.add(file);
}
}
}
}

1
src/main/java/io/linuxserver/davos/schedule/workflow/ScheduleWorkflow.java
View File

@ -16,7 +16,6 @@ public class ScheduleWorkflow {
private Connection connection;
public ScheduleWorkflow(ScheduleConfiguration config) {
this.config = config;
}

15
src/main/java/io/linuxserver/davos/schedule/workflow/transfer/FileRecursiveTransferStrategy.java Normal file
View File

@ -0,0 +1,15 @@
package io.linuxserver.davos.schedule.workflow.transfer;
import io.linuxserver.davos.transfer.ftp.connection.Connection;
public class FileRecursiveTransferStrategy extends TransferStrategy {
public FileRecursiveTransferStrategy(Connection connection) {
super(connection);
}
@Override
public void transfer(String from, String to) {
// TODO Auto-generated method stub
}
}

15
src/main/java/io/linuxserver/davos/schedule/workflow/transfer/FilesAndFoldersTranferStrategies.java Normal file
View File

@ -0,0 +1,15 @@
package io.linuxserver.davos.schedule.workflow.transfer;
import io.linuxserver.davos.transfer.ftp.connection.Connection;
public class FilesAndFoldersTranferStrategies extends TransferStrategy {
public FilesAndFoldersTranferStrategies(Connection connection) {
super(connection);
}
@Override
public void transfer(String from, String to) {
// TODO Auto-generated method stub
}
}

15
src/main/java/io/linuxserver/davos/schedule/workflow/transfer/FilesOnlyTransferStrategy.java Normal file
View File

@ -0,0 +1,15 @@
package io.linuxserver.davos.schedule.workflow.transfer;
import io.linuxserver.davos.transfer.ftp.connection.Connection;
public class FilesOnlyTransferStrategy extends TransferStrategy {
public FilesOnlyTransferStrategy(Connection connection) {
super(connection);
}
@Override
public void transfer(String from, String to) {
// TODO Auto-generated method stub
}
}

14
src/main/java/io/linuxserver/davos/schedule/workflow/transfer/TransferStrategy.java Normal file
View File

@ -0,0 +1,14 @@
package io.linuxserver.davos.schedule.workflow.transfer;
import io.linuxserver.davos.transfer.ftp.connection.Connection;
public abstract class TransferStrategy {
protected Connection connection;
public TransferStrategy(Connection connection) {
this.connection = connection;
}
public abstract void transfer(String from, String to);
}

36
src/main/java/io/linuxserver/davos/transfer/ftp/FTPFile.java
View File

@ -1,5 +1,41 @@
package io.linuxserver.davos.transfer.ftp;
import org.joda.time.DateTime;
public class FTPFile {
private String name;
private long size;
private String absolutePath;
private DateTime lastModified;
private boolean directory;
public FTPFile(String name, long size, String absolutePath, long mTime, boolean directory) {
this.name = name;
this.size = size;
this.absolutePath = absolutePath;
this.lastModified = new DateTime(mTime);
this.directory = directory;
}
public String getName() {
return name;
}
public long getSize() {
return size;
}
public String getAbsolutePath() {
return absolutePath;
}
public DateTime getLastModified() {
return lastModified;
}
public boolean isDirectory() {
return directory;
}
}

5
src/main/java/io/linuxserver/davos/transfer/ftp/FileTransferType.java Normal file
View File

@ -0,0 +1,5 @@
package io.linuxserver.davos.transfer.ftp;
public enum FileTransferType {
FILES_ONLY, FILE_RECURSIVE, INCLUDE_FOLDERS;
}

9
src/main/java/io/linuxserver/davos/transfer/ftp/connection/Connection.java
View File

@ -3,14 +3,15 @@ package io.linuxserver.davos.transfer.ftp.connection;
import java.util.List;
import io.linuxserver.davos.transfer.ftp.FTPFile;
import io.linuxserver.davos.transfer.ftp.exception.FTPException;
public interface Connection {
String currentDirectory();
String currentDirectory() throws FTPException;
void download(String remoteFilePath, String localFilePath);
void download(String remoteFilePath, String localFilePath) throws FTPException;
List<FTPFile> listFiles();
List<FTPFile> listFiles() throws FTPException;
List<FTPFile> listFiles(String remoteDirectory);
List<FTPFile> listFiles(String remoteDirectory) throws FTPException;
}

18
src/main/java/io/linuxserver/davos/transfer/ftp/exception/FileListingException.java Normal file
View File

@ -0,0 +1,18 @@
package io.linuxserver.davos.transfer.ftp.exception;
public class FileListingException extends FTPException {
private static final long serialVersionUID = 7733358928451506618L;
public FileListingException() {
super();
}
public FileListingException(String message) {
super(message);
}
public FileListingException(String message, Exception cause) {
super(message, cause);
}
}

8
src/main/java/io/linuxserver/davos/util/PatternBuilder.java Normal file
View File

@ -0,0 +1,8 @@
package io.linuxserver.davos.util;
public class PatternBuilder {
public static String buildFromFilterString(String filter) {
return filter.replaceAll("\\?", ".{1}").replaceAll("\\*", ".+");
}
}

218
src/test/java/io/linuxserver/davos/schedule/workflow/FilterFilesWorkflowStepTest.java Normal file
View File

@ -0,0 +1,218 @@
package io.linuxserver.davos.schedule.workflow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.initMocks;
import java.util.ArrayList;
import java.util.Arrays;
import org.joda.time.DateTime;
import org.junit.Before;
import org.junit.Test;
import org.mockito.InOrder;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import io.linuxserver.davos.schedule.ScheduleConfiguration;
import io.linuxserver.davos.transfer.ftp.FTPFile;
import io.linuxserver.davos.transfer.ftp.connection.Connection;
import io.linuxserver.davos.transfer.ftp.exception.FileListingException;
public class FilterFilesWorkflowStepTest {
@InjectMocks
private FilterFilesWorkflowStep workflowStep = new FilterFilesWorkflowStep();
@Mock
private DownloadFilesWorkflowStep mockNextStep;
@Mock
private Connection mockConnection;
@Before
public void setUp() {
initMocks(this);
}
@Test
public void workflowStepShouldListFilesInTheRemoteDirectory() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/");
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
schedule.setConnection(mockConnection);
workflowStep.runStep(schedule);
verify(mockConnection).listFiles("remote/");
}
@Test
public void workflowStepShouldFilterOutAnyFilesThatAreNotInTheGivenConfigList() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/");
config.setFilters(Arrays.asList("file1", "file2", "file4"));
config.setLastRun(DateTime.now().minusDays(1));
ArrayList<FTPFile> files = new ArrayList<FTPFile>();
FTPFile file1 = new FTPFile("file1", 0, "remote/", DateTime.now().getMillis(), false);
FTPFile file2 = new FTPFile("file2", 0, "remote/", DateTime.now().getMillis(), false);
FTPFile file3 = new FTPFile("file3", 0, "remote/", DateTime.now().getMillis(), false);
FTPFile file4 = new FTPFile("file4", 0, "remote/", DateTime.now().getMillis(), false);
FTPFile file5 = new FTPFile("file5", 0, "remote/", DateTime.now().getMillis(), false);
files.add(file1);
files.add(file2);
files.add(file3);
files.add(file4);
files.add(file5);
when(mockConnection.listFiles("remote/")).thenReturn(files);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
schedule.setConnection(mockConnection);
workflowStep.runStep(schedule);
verify(mockNextStep).setFilesToDownload(Arrays.asList(file1, file2, file4));
}
@Test
public void workflowStepShouldFilterOutAnyFilesThatAreNotInTheGivenConfigListAndWereModifiedBeforeLastRun() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/");
config.setFilters(Arrays.asList("file1", "file2", "file4"));
config.setLastRun(DateTime.now().minusDays(1));
ArrayList<FTPFile> files = new ArrayList<FTPFile>();
FTPFile file1 = new FTPFile("file1", 0, "remote/", DateTime.now().minusDays(2).getMillis(), false);
FTPFile file2 = new FTPFile("file2", 0, "remote/", DateTime.now().getMillis(), false);
FTPFile file3 = new FTPFile("file3", 0, "remote/", DateTime.now().minusDays(2).getMillis(), false);
FTPFile file4 = new FTPFile("file4", 0, "remote/", DateTime.now().minusDays(2).getMillis(), false);
FTPFile file5 = new FTPFile("file5", 0, "remote/", DateTime.now().getMillis(), false);
files.add(file1);
files.add(file2);
files.add(file3);
files.add(file4);
files.add(file5);
when(mockConnection.listFiles("remote/")).thenReturn(files);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
schedule.setConnection(mockConnection);
workflowStep.runStep(schedule);
verify(mockNextStep).setFilesToDownload(Arrays.asList(file2));
}
@Test
public void workflowStepShouldFilterOutAnyFilesThatDoNotMatchTheWildcards() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/");
config.setFilters(Arrays.asList("file1?and?Stuff", "file2*something", "file4*", "file5"));
config.setLastRun(DateTime.now().minusDays(1));
ArrayList<FTPFile> files = new ArrayList<FTPFile>();
FTPFile file1 = new FTPFile("file1.and-stuff", 0, "remote/", DateTime.now().minusDays(2).getMillis(), false);
FTPFile file2 = new FTPFile("file2.andMoreTextsomething", 0, "remote/", DateTime.now().getMillis(), false);
FTPFile file3 = new FTPFile("file3", 0, "remote/", DateTime.now().minusDays(2).getMillis(), false);
FTPFile file4 = new FTPFile("file4.txt", 0, "remote/", DateTime.now().getMillis(), false);
FTPFile file5 = new FTPFile("file5.txt", 0, "remote/", DateTime.now().getMillis(), false);
files.add(file1);
files.add(file2);
files.add(file3);
files.add(file4);
files.add(file5);
when(mockConnection.listFiles("remote/")).thenReturn(files);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
schedule.setConnection(mockConnection);
workflowStep.runStep(schedule);
verify(mockNextStep).setFilesToDownload(Arrays.asList(file2, file4));
}
@Test
public void workflowStepShouldCallNextStepRunMethodOnceSettingFilters() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/");
config.setFilters(Arrays.asList("file1", "file2", "file4"));
config.setLastRun(DateTime.now().minusDays(1));
ArrayList<FTPFile> files = new ArrayList<FTPFile>();
FTPFile file1 = new FTPFile("file1", 0, "remote/", DateTime.now().getMillis(), false);
files.add(file1);
when(mockConnection.listFiles("remote/")).thenReturn(files);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
schedule.setConnection(mockConnection);
workflowStep.runStep(schedule);
InOrder inOrder = Mockito.inOrder(mockNextStep);
inOrder.verify(mockNextStep).setFilesToDownload(Arrays.asList(file1));
inOrder.verify(mockNextStep).runStep(schedule);
}
@Test
public void ifFilterListIsInitiallyEmptyThenAssumeThatAllFilesAfterLastRunShouldBeDownloaded() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/");
config.setLastRun(DateTime.now().minusDays(1));
ArrayList<FTPFile> files = new ArrayList<FTPFile>();
FTPFile file1 = new FTPFile("file1", 0, "remote/", DateTime.now().minusDays(2).getMillis(), false);
FTPFile file2 = new FTPFile("file2", 0, "remote/", DateTime.now().getMillis(), false);
FTPFile file3 = new FTPFile("file3", 0, "remote/", DateTime.now().minusDays(2).getMillis(), false);
FTPFile file4 = new FTPFile("file4", 0, "remote/", DateTime.now().minusDays(2).getMillis(), false);
FTPFile file5 = new FTPFile("file5", 0, "remote/", DateTime.now().getMillis(), false);
files.add(file1);
files.add(file2);
files.add(file3);
files.add(file4);
files.add(file5);
when(mockConnection.listFiles("remote/")).thenReturn(files);
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
schedule.setConnection(mockConnection);
workflowStep.runStep(schedule);
verify(mockNextStep).setFilesToDownload(Arrays.asList(file2, file5));
}
@Test
public void ifListingFilesIsUnsuccessfulThenDoNotCallNextStep() {
ScheduleConfiguration config = new ScheduleConfiguration(null, null, null, 0, null, "remote/", "local/");
when(mockConnection.listFiles("remote/")).thenThrow(new FileListingException());
ScheduleWorkflow schedule = new ScheduleWorkflow(config);
schedule.setConnection(mockConnection);
workflowStep.runStep(schedule);
verify(mockNextStep, never()).runStep(schedule);
}
}

44
src/test/java/io/linuxserver/davos/util/PatternBuilderTest.java Normal file
View File

@ -0,0 +1,44 @@
package io.linuxserver.davos.util;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.Test;
public class PatternBuilderTest {
@Test
public void builderShouldTurnQuestionMarksIntoSingleCharacterRegexMatcher() {
String filter = "This?is?a filter";
String expected = "This.{1}is.{1}a filter";
assertThat(PatternBuilder.buildFromFilterString(filter)).isEqualTo(expected);
}
@Test
public void builderShouldTurnAsterixesIntoManyCharacterRegexMatcher() {
String filter = "This*is*a filter";
String expected = "This.+is.+a filter";
assertThat(PatternBuilder.buildFromFilterString(filter)).isEqualTo(expected);
}
@Test
public void regexStringReturnedShouldBeAbleToActuallyMatchUsingRegexOperation() {
String normalValue = "Clean Code.pdf";
String filteredValue = "Clean?Code*";
assertThat(normalValue.matches(PatternBuilder.buildFromFilterString(filteredValue))).isTrue();
}
@Test
public void stringWithBothAsterixAndQuestionMarkShouldMatchProperly() {
String anotherValue = "File Name with a Prefix12Then some text";
String slightlyMoreComplicated = "File?Name*Prefix??Then some text";
assertThat(anotherValue.matches(PatternBuilder.buildFromFilterString(slightlyMoreComplicated))).isTrue();
}
}