Skip to content

Allow chainsaw to read a directory containing more than one file #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Allow chainsaw to read a directory containing more than one file #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5c25131
Create a function to process a single file.
tzachi-dar Jan 31, 2019
f01ead3
Allow to have the input as a pattern of files (for example c:\dir\fil…
tzachi-dar Feb 3, 2019
52457c3
Move to use commons-vfs2 (Increase stability of logger).
tzachi-dar Feb 4, 2019
e0f3963
Do not display the user name and password in the log and in the ui.
tzachi-dar Feb 14, 2019
b079c70
Use maxLogingDays to limit the number of files that will be read.
tzachi-dar Mar 12, 2019
6b63c22
Fix a bug that caused that when reading gzip files, they have been re…
tzachi-dar Mar 13, 2019
22ee868
Update commons-vfs2 to it's latest version.
tzachi-dar Mar 13, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,10 +390,10 @@
<artifactId>xstream</artifactId>
<version>1.1.2</version>
</dependency>
<dependency>
<groupId>commons-vfs</groupId>
<artifactId>commons-vfs</artifactId>
<version>1.0</version>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-vfs2</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
Expand Down Expand Up @@ -437,6 +437,11 @@
<version>1.16.20</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
</dependency>
</dependencies>

<reporting>
Expand Down
172 changes: 154 additions & 18 deletions src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,23 @@

package org.apache.log4j.chainsaw.vfs;

import org.apache.commons.vfs.*;
import org.apache.commons.vfs.provider.URLFileName;
import org.apache.commons.vfs.provider.sftp.SftpFileSystemConfigBuilder;
import org.apache.commons.vfs.util.RandomAccessMode;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.vfs2.*;
import org.apache.commons.vfs2.provider.URLFileName;
import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
import org.apache.commons.vfs2.util.RandomAccessMode;
import org.apache.log4j.chainsaw.receivers.VisualReceiver;
import org.apache.log4j.varia.LogFilePatternReceiver;

import javax.swing.*;
import java.awt.*;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.ListIterator;
import java.util.Vector;
import java.util.zip.GZIPInputStream;

/**
Expand Down Expand Up @@ -145,6 +152,7 @@
public class VFSLogFilePatternReceiver extends LogFilePatternReceiver implements VisualReceiver {

private boolean promptForUserInfo = false;
private String maxLogingDays = null;
private Container container;
private final Object waitForContainerLock = new Object();
private boolean autoReconnect;
Expand Down Expand Up @@ -178,6 +186,37 @@ public boolean isPromptForUserInfo() {
return promptForUserInfo;
}


/**
* Use maxLogingDays to limit the loging to recent messages.
* For example to see only data from 10 days, set this parameter do 10.
*
* @param maxLogingDays Don't open log files that are older than maxLogingDays
*/
public void setMaxLogingDays(String maxLogingDays) {
this.maxLogingDays = maxLogingDays;
}

public String getMaxLogingDays() {
return maxLogingDays;
}

private Long getStartLogingTime() {
if (getMaxLogingDays() == null) {
return 0L;
}
try {
long days = Integer.parseInt(getMaxLogingDays());
getLogger().info("MaxloggingDays that will be used is " + days);
return System.currentTimeMillis() - days * 3600 * 24 * 1000;
}
catch (NumberFormatException e) {
getLogger().error("exception parding MaxloggingDays " + getMaxLogingDays(),e);
return 0L;
}
// Can not reach here.
}

/**
* Accessor
*
Expand Down Expand Up @@ -318,36 +357,132 @@ private class VFSReader implements Runnable {
private boolean isGZip(String fileName) {
return fileName.endsWith( ".gz" );
}

private String getHostName(FileName fileName) {
if (fileName instanceof URLFileName) {
return ((URLFileName)fileName).getHostName();
}
return "unknown";
}

// Try to sort the files based on the time that they have been created.
// We would like to read them from the latest and then continue to the newest, which we will also tail.
private void Sort(Vector<FileObject> filesToRead) {
if (filesToRead.size() == 0) {
return;
}
Collections.sort(filesToRead, new Comparator<FileObject>() {
@Override
public int compare(FileObject o1, FileObject o2) {
try {
return Long.compare(o1.getContent().getLastModifiedTime(),
o2.getContent().getLastModifiedTime());
} catch (FileSystemException fse) {
getLogger().error("Error, could not get file time", fse);
return 0;
}
}
});

getLogger().info("Going to read the following files");
SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy");

try {
Long StartTime = getStartLogingTime();
for (ListIterator<FileObject> iter = filesToRead.listIterator(); iter.hasNext(); ) {
FileObject f =iter.next();
boolean ignoring = false;
String dates = sdf.format(new Date(f.getContent().getLastModifiedTime()));
if(f.getContent().getLastModifiedTime() < StartTime) {
iter.remove();
ignoring = true;
}
getLogger().info(
"file = " + f.getName().getBaseName() + " last modified time " + dates + (ignoring ? " will not be read because of MaxLogingDays" : ""));

}
} catch (FileSystemException fse) {
getLogger().error("Error, could not get file time", fse);
}
}

public void run() {
Vector<FileObject> filesToRead = new Vector<FileObject>();
getLogger().info("starting to read " + getFileURL());
if (!getFileURL().contains("*")) {
ProcessSingleFile(getFileURL(), null, null, false);
return;
}

int atIndex = getFileURL().indexOf("@");
int protocolIndex = getFileURL().indexOf("://");
String loggableFileURL = atIndex > -1 ? getFileURL().substring(0, protocolIndex + "://".length()) + "username:password" + getFileURL().substring(atIndex) :getFileURL();
getLogger().info("starting to read loggableFileURL" + loggableFileURL);

String dir = FilenameUtils.getFullPathNoEndSeparator(getFileURL());
String baseName = FilenameUtils.getName(getFileURL());
getLogger().debug("dir = " + dir + " name " + baseName);

try {
FileSystemManager fileSystemManager = VFS.getManager();
FileSystemOptions opts = new FileSystemOptions();

try {
SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no");
SftpFileSystemConfigBuilder.getInstance( ).setUserDirIsRoot(opts, false);
} catch (NoClassDefFoundError ncdfe) {
getLogger().warn("JSch not on classpath!", ncdfe);
}

FileObject localFileObject = fileSystemManager.resolveFile(dir, opts);
String dirName = localFileObject.getName().getPath();

FileObject[] children = localFileObject.getChildren();
for (int i = 0; i < children.length; i++) {
if (FilenameUtils.wildcardMatchOnSystem(children[i].getName().getBaseName(), baseName)) {
filesToRead.add(children[i]);
}
}
Sort(filesToRead);
for (int i = 0; i < filesToRead.size(); i++) {
ProcessSingleFile(filesToRead.get(i).getName().toString(), getHostName(filesToRead.get(i).getName()),//urlFileName.getHostName(),
dirName + FileName.SEPARATOR + baseName, i != filesToRead.size() - 1);
}

} catch (FileSystemException fse) {
getLogger().info("Error processing directory " + loggableFileURL + " exiting", fse);
}
}

private void ProcessSingleFile(String fileUrl, String hostName, String cannonicalPath, boolean forceNoTail) {
Reader reader = null;
FileObject fileObject = null;
//thread should end when we're no longer active
while (reader == null && !terminated) {
int atIndex = getFileURL().indexOf("@");
int protocolIndex = getFileURL().indexOf("://");
int atIndex =fileUrl.indexOf("@");
int protocolIndex = fileUrl.indexOf("://");

String loggableFileURL = atIndex > -1 ? getFileURL().substring(0, protocolIndex + "://".length()) + "username:password" + getFileURL().substring(atIndex) : getFileURL();
String loggableFileURL = atIndex > -1 ? fileUrl.substring(0, protocolIndex + "://".length()) + "username:password" + fileUrl.substring(atIndex) :fileUrl;
getLogger().info("attempting to load file: " + loggableFileURL);
try {
FileSystemManager fileSystemManager = VFS.getManager();
FileSystemOptions opts = new FileSystemOptions();
//if jsch not in classpath, can get NoClassDefFoundError here
try {
SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no");
SftpFileSystemConfigBuilder.getInstance( ).setUserDirIsRoot(opts, false);
} catch (NoClassDefFoundError ncdfe) {
getLogger().warn("JSch not on classpath!", ncdfe);
}

synchronized (fileSystemManager) {
fileObject = fileSystemManager.resolveFile(getFileURL(), opts);
fileObject = fileSystemManager.resolveFile(fileUrl, opts);
if (fileObject.exists()) {
reader = new InputStreamReader(fileObject.getContent().getInputStream(), "UTF-8");
//now that we have a reader, remove additional portions of the file url (sftp passwords, etc.)
//check to see if the name is a URLFileName..if so, set file name to not include username/pass
if (fileObject.getName() instanceof URLFileName) {
URLFileName urlFileName = (URLFileName) fileObject.getName();
setHost(urlFileName.getHostName());
setPath(urlFileName.getPath());
}
setHost(hostName != null ? hostName : getHostName(fileObject.getName()) );
setPath(cannonicalPath != null ? cannonicalPath : fileObject.getName().getPath());
} else {
getLogger().info(loggableFileURL + " not available - will re-attempt to load after waiting " + MISSING_FILE_RETRY_MILLIS + " millis");
}
Expand Down Expand Up @@ -385,6 +520,7 @@ public void run() {
//if jsch not in classpath, can get NoClassDefFoundError here
try {
SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no");
SftpFileSystemConfigBuilder.getInstance( ).setUserDirIsRoot(opts, false);
} catch (NoClassDefFoundError ncdfe) {
getLogger().warn("JSch not on classpath!", ncdfe);
}
Expand All @@ -393,11 +529,10 @@ public void run() {
synchronized (fileSystemManager) {
if (fileObject != null) {
fileObject.getFileSystem().getFileSystemManager().closeFileSystem(fileObject.getFileSystem());
fileObject.close();
fileObject = null;
}

fileObject = fileSystemManager.resolveFile(getFileURL(), opts);
fileObject = fileSystemManager.resolveFile(fileUrl, opts);
}

//file may not exist..
Expand All @@ -410,12 +545,13 @@ public void run() {
getLogger().info(getPath() + " - unable to refresh fileobject", err);
}

if (isGZip(getFileURL())) {
if (isGZip(fileUrl)) {
InputStream gzipStream = new GZIPInputStream(fileObject.getContent().getInputStream());
Reader decoder = new InputStreamReader(gzipStream, "UTF-8");
BufferedReader bufferedReader = new BufferedReader(decoder);
process(bufferedReader);
readingFinished = true;
continue;
}
//could have been truncated or appended to (don't do anything if same size)
if (fileObject.getContent().getSize() < lastFileSize) {
Expand Down Expand Up @@ -455,7 +591,7 @@ public void run() {
if (isTailing() && fileLarger && !terminated) {
getLogger().debug(getPath() + " - tailing file - file size: " + lastFileSize);
}
} while (isTailing() && !terminated && !readingFinished);
} while (isTailing() && !terminated && !readingFinished && !forceNoTail);
} catch (IOException ioe) {
getLogger().info(getPath() + " - exception processing file", ioe);
try {
Expand All @@ -472,7 +608,7 @@ public void run() {
} catch (InterruptedException ie) {
}
}
} while (isAutoReconnect() && !terminated && !readingFinished);
} while (isAutoReconnect() && !terminated && !readingFinished && !forceNoTail);
getLogger().debug(getPath() + " - processing complete");
}

Expand Down
2 changes: 2 additions & 0 deletions src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiverBeanInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ public PropertyDescriptor[] getPropertyDescriptors() {
"filterExpression", VFSLogFilePatternReceiver.class),
new PropertyDescriptor(
"promptForUserInfo", VFSLogFilePatternReceiver.class),
new PropertyDescriptor(
"maxLogingDays", VFSLogFilePatternReceiver.class),
new PropertyDescriptor("group", VFSLogFilePatternReceiver.class),
};
} catch (Exception e) {
Expand Down