最近有个项目，是硬件结合的，硬件上传到服务器的日志，每天数百万条，有时候某个设备出问题了，因为日志的数据很混乱，很难查出具体的原因。
所以写了这个工具，主要是提高日志分析的效率，可以通过关键词提取日志数据。
工具使用了多线程、I/O等技术，本人技术有限，所以只能写到这样子，测试过很多次。
测试出来的数据：400MB的日志，5个线程：96~97秒完成分割，分割出来的日志大小大同小异，为什么不把分割出来的日志合并呢？因为线程的启动时间不是顺序的，加上本人懒，所以没做了。
不建议使用超过20个线程去处理日志。因为如果是2GB的数据，10个线程去处理，每个线程也只需要处理204.8MB。这个已经是非常快的效率了。
原理：
java源码：
package test;
import io.netty.util.concurrent.DefaultThreadFactory;
import javax.swing.*;
import javax.swing.filechooser.FileFilter;
import java.awt.*;
import java.awt.event.*;
import java.io.*;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.LinkedList;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public class WebLogParser {
public static LinkedBlockingQueue<String> logQueue = new LinkedBlockingQueue<>();//加入队列写出日志
public static ThreadPoolExecutor executor;
public static JTextField filePath, threadNum, keywords;
public static JTextArea logs;
public static JButton start, stop, choose;
public static JLabel text;
public static WorkerListener listener;
public static Timer timer;
public static boolean isStop = false;
public static double processLen = 0.0;
public static double splitLenSum = 0;
public static double fileLen = 0.0;
public static File file;
public static DefaultThreadFactory factory;
private static String keywordsTip = "you can use \";\" split this string.";
public static void main(String[] args) {
// TODO Auto-generated method stub
JFrame windows = createWindows("日志分析", windowListener());
createPanel(windows);
ActionListener actionListener = createClickListener();
start.addActionListener(actionListener);
stop.addActionListener(actionListener);
choose.addActionListener(actionListener);
filePath.setText("C:\\Users\\Administrator\\Desktop\\1.log");
// keywords.setText("867186033556969");
threadNum.setText("1");
windows.setBounds(100, 100, 700, 480);
windows.setVisible(true);
StringBuffer sb = new StringBuffer();
timer = new Timer();
timer.schedule(new TimerTask() {
@Override
public void run() {
StringBuffer sb = new StringBuffer();
if (executor != null) {
sb.append("核心线程:" + executor.getCorePoolSize() + " ")
.append("激活线程:" + executor.getActiveCount() + " ");
}
if (processLen > 0 || splitLenSum > 0) {
if (processLen > 0) {
text.setText("搜索进度：" + Math.ceil(processLen / fileLen * 100) + "% - " + nowDateTime() + " - " + sb.toString());
if (Math.ceil(processLen / fileLen * 100) == 100) {
initView(true);
}
} else {
text.setText("分割进度：" + formatNumber(splitLenSum / fileLen * 100, null) + "% - " + nowDateTime() + " - " + sb.toString());
}
} else {
text.setText(String.format("init success. - " + sb.toString()));
}
}
}, 1000, 1000);
initView(true);
}
private static void initView(boolean isEnable) {
out("init view.");
if (isEnable) {
isStop = true;
start.setEnabled(true);
stop.setEnabled(false);
threadNum.setEnabled(true);
keywords.setEnabled(true);
filePath.setEnabled(true);
} else {
isStop = false;
start.setEnabled(false);
stop.setEnabled(true);
threadNum.setEnabled(false);
keywords.setEnabled(false);
filePath.setEnabled(false);
}
processLen = 0;
splitLenSum = 0;
}
private static void doWork() {
factory = new DefaultThreadFactory("logs");
factory.newThread(new Runnable() {
@Override
public void run() {
while (!isStop){
if (logQueue.size()>0){
try {
logs.setText(logQueue.take()+"\n" + logs.getText());
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
}).start();
int tnum = Integer.valueOf(threadNum.getText());
if (file == null){
file = new File(filePath.getText());
}
out("read file " + file.getName());
String key = keywords.getText();
if (key.contains(keywordsTip)) {
key = null;
}
listener = createWorkerListener();
out("Create Thread...");
if (executor == null) {
executor = new ThreadPoolExecutor(20, 20, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100));
}
for (int i = 0; i < tnum; i++) {
executor.execute(new Worker(file, i, key, null, logQueue, listener, tnum));
}
out("Create Thread Success...");
}
private static WorkerListener createWorkerListener() {
return new WorkerListener() {
@Override
public void process(long f, long process, long pos) {
processLen = process;
fileLen = f;
}
@Override
public void splitFile(long fl, long splitLen, long len) {
splitLenSum = splitLen;
fileLen = fl;
}
@Override
public boolean stop() {
return isStop;
}
};
}
private static ActionListener createClickListener() {
return new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
if (e.getActionCommand().equals("start")) {
if ("".equals(filePath.getText()) || filePath.getText() == null) {
JOptionPane.showMessageDialog(null, "请选择日志文件路径", "info", JOptionPane.PLAIN_MESSAGE);
return;
}
if ("".equals(threadNum.getText()) || threadNum.getText() == null) {
JOptionPane.showMessageDialog(null, "请输入线程数", "info", JOptionPane.PLAIN_MESSAGE);
return;
}
if (Integer.valueOf(threadNum.getText()) <= 0) {
JOptionPane.showMessageDialog(null, "线程数必须>0", "info", JOptionPane.PLAIN_MESSAGE);
return;
}
if (Integer.valueOf(threadNum.getText()) > 20) {
JOptionPane.showMessageDialog(null, "线程数必须<=20", "info", JOptionPane.PLAIN_MESSAGE);
return;
}
if ("".equals(keywords.getText()) || keywords.getText() == null) {
JOptionPane.showMessageDialog(null, "请输入关键词", "info", JOptionPane.PLAIN_MESSAGE);
return;
}
initView(false);
doWork();
} else if (e.getActionCommand().equals("stop")) {
initView(true);
} else if (e.getActionCommand().equals(">>")) {
chooseFile();
} else {
out(e.getActionCommand());
}
}
};
}
private static void chooseFile() {
JFileChooser jfc = new JFileChooser();
jfc.addChoosableFileFilter(new FileFilter() {
@Override
public boolean accept(File f) {
String name = f.getName();
return f.isDirectory() || name.toLowerCase().endsWith(".log");
}
@Override
public String getDescription() {
return "*.log";
}
});
jfc.showDialog(new JLabel(), "选择日志文件");
jfc.setFileSelectionMode(JFileChooser.FILES_ONLY);
file = jfc.getSelectedFile();
if (file != null && file.isFile()) {
filePath.setText(file.getAbsolutePath() + file.getName());
}
}
private static WindowListener windowListener() {
return new WindowAdapter() {
@Override
public void windowOpened(WindowEvent e) {
super.windowOpened(e);
}
@Override
public void windowClosing(WindowEvent e) {
isStop = true;
timer.cancel();
super.windowClosing(e);
if (executor != null) {
executor.shutdownNow();
}
}
@Override
public void windowClosed(WindowEvent e) {
super.windowClosed(e);
if (executor != null) {
executor.shutdown();
}
}
@Override
public void windowActivated(WindowEvent e) {
super.windowActivated(e);
}
@Override
public void windowStateChanged(WindowEvent e) {
super.windowStateChanged(e);
}
};
}
private static JFrame createWindows(String title, WindowListener listener) {
JFrame jFrame = new JFrame(title);
jFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
jFrame.addWindowListener(listener);
return jFrame;
}
private static JPanel createPanel(JFrame jFrame) {
JPanel jPanel = new JPanel();
jPanel.setLayout(null);
jFrame.add(jPanel);
/**
* lable
*/
JLabel label1 = new JLabel("Log File Absolute Path:", JLabel.RIGHT);
label1.setBounds(20, 20, 140, 26);
JLabel label2 = new JLabel("Thread Number:", JLabel.RIGHT);
label2.setBounds(20, 56, 140, 26);
JLabel label4 = new JLabel("keywords:", JLabel.RIGHT);
label4.setBounds(20, 92, 140, 26);
JLabel label3 = new JLabel("logs:", JLabel.RIGHT);
label3.setBounds(4, 132, 48, 26);
label3.setBackground(Color.BLACK);
text = new JLabel();
text.setBounds(20, 400, 500, 26);
text.setText("init success.");
text.setForeground(Color.gray);
/**
* editor
*/
filePath = new JTextField();
filePath.setBounds(165, 20, 400, 26);
choose = new JButton(">>");
choose.setBounds(565, 20, 26, 25);
threadNum = new JTextField();
threadNum.setBounds(165, 56, 80, 26);
threadNum.addKeyListener(new KeyListener() {
@Override
public void keyTyped(KeyEvent e) {
int temp = e.getKeyChar();
if (temp == 10) {//按回车时
} else if (temp != 46) {
if (temp != 8) {
if (temp > 57) {
e.consume();
} else if (temp < 48) {
e.consume();
}
}
}
}
@Override
public void keyPressed(KeyEvent e) {
}
@Override
public void keyReleased(KeyEvent e) {
}
});
keywords = new JTextField();
keywords.setBounds(165, 92, 495, 26);
keywords.setText(keywordsTip);
keywords.setForeground(Color.gray);
keywords.addFocusListener(new FocusAdapter() {
@Override
public void focusGained(FocusEvent e) {
super.focusGained(e);
if (keywords.getText().contains(keywordsTip)) {
keywords.setText("");
}
keywords.setForeground(Color.black);
}
@Override
public void focusLost(FocusEvent e) {
super.focusLost(e);
if ("".equals(keywords.getText())) {
keywords.setForeground(Color.gray);
keywords.setText(keywordsTip);
}
}
});
logs = new JTextArea(10, 20);
logs.setBounds(60, 130, 600, 200);
logs.setLineWrap(true);
JScrollPane logSp = new JScrollPane(logs);
logSp.setBounds(60, 130, 600, 200);
/**
* button
*/
start = new JButton("start");
start.setBounds(580, 360, 65, 26);
stop = new JButton("stop");
stop.setBounds(500, 360, 65, 26);
jPanel.add(label1);
jPanel.add(label2);
jPanel.add(label3);
jPanel.add(label4);
jPanel.add(text);
jPanel.add(filePath);
jPanel.add(choose);
jPanel.add(threadNum);
jPanel.add(keywords);
// jPanel.add(logs);
jPanel.add(logSp);
jPanel.add(start);
jPanel.add(stop);
return jPanel;
}
private static void out(String s) {
if (s != null) {
s = nowDateTime() + " " + s;
System.out.println(s);
logs.setText(s + "\n" + logs.getText());
}
}
private static String nowDateTime() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
return sdf.format(new Date());
}
private static String formatNumber(Double number, String pattern) {
if (pattern == null) {
pattern = "#0.00";
}
DecimalFormat df = new DecimalFormat(pattern);
return df.format(number);
}
}
class Worker implements Runnable {
private static final boolean debug = true;
private File file;
/**
* index must be start in 0;
*/
private long index = 0;
private int threadNum = 1;
/**
* you can use ";" split this string.
*/
private String keywords = null;
private File targetFile = null;
private FileOutputStream fos = null;
private LinkedBlockingQueue<String> logQueue;
private long stime = 0L;
long pos = 0L;
private WorkerListener listener;
public Worker() {
// TODO Auto-generated constructor stub
}
public Worker(File f) {
this(f, 1L);
}
public Worker(File f, long num) {
this(f, num, null);
}
public Worker(File file, long index, String keywords) {
this(file, index, keywords, null);
}
public Worker(File file, long index, String keywords, File targetFile) {
this(file, index, keywords, targetFile, null, null);
}
public Worker(File file, long index, String keywords, File targetFile, LinkedBlockingQueue<String> logQueue) {
this(file, index, keywords, targetFile, logQueue, null);
}
public Worker(File file, long index, String keywords, File targetFile, LinkedBlockingQueue<String> logQueue, WorkerListener listener) {
this(file, index, keywords, targetFile, logQueue, listener, 1);
}
public Worker(File file, long index, String keywords, File targetFile, LinkedBlockingQueue<String> logQueue, WorkerListener listener, int threadNum) {
this.file = file;
this.index = index;
this.keywords = keywords;
this.targetFile = targetFile;
this.logQueue = logQueue;
this.listener = listener;
this.threadNum = threadNum;
}
@Override
public void run() {
// TODO Auto-generated method stub
out("file exists: "+file.exists());
if (file.exists()) {
stime = System.currentTimeMillis();
out("Thread start....");
this.checkTargetFile();
RandomAccessFile raf = null;
try {
long len = 0L;
if (index > 0) {
pos = (long) Math.floor(file.length() / (index + 1));
}
file = splitFiles(file);
raf = new RandomAccessFile(file, "r");
raf.seek(pos);
String line;
out("init point is " + pos);
out("temp file is " + file.getAbsolutePath() + file.getName());
while ((line = raf.readLine()) != null) {
len += line.length();
if (listener != null) {
if (listener.stop()) {
break;
}
listener.process(file.length(), len, pos);
}
if (keywords != null) {
if (this.checkKeywords(line)) {
this.put(line);
}
}
}
out(String.format("search end and file length is %s,search total is %s.", file.length(), len));
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (raf != null) {
raf.close();
}
if (fos != null) {
fos.close();
}
} catch (IOException e) {
e.printStackTrace();
}
stime = (System.currentTimeMillis() - stime) / 1000;
out("Thread end,The work use time is " + stime + "s.");
delSplitFile(file);
}
}
}
private void delSplitFile(File file) {
if (threadNum > 1) {
file.delete();
}
}
private File splitFiles(File file) throws Exception {
if (threadNum < 1) {
throw new Exception("Thread number must be >= 1.");
}
if (threadNum == 1) {
return file;
}
long len = (file.length() / threadNum);
File newFile = new File(getFilePath() + getFileName() + Thread.currentThread().getName());
FileInputStream _in = new FileInputStream(file);
FileOutputStream _out = new FileOutputStream(newFile);
out("split file and skip " + pos + ",split length " + len);
_in.skip(pos);
byte[] b = new byte[1024];
int n = 0;
long l = 0;
stime = System.currentTimeMillis();
out("start split file.");
while ((n = _in.read(b)) != -1) {
l += n;
if (listener != null) {
if (listener.stop()) {
break;
}
listener.splitFile(len, l, n);
}
// out(String.format(">>test>>fileLen:%s sum:%s now:%s",len,l,n));
if (l > len) {
break;
}
_out.write(b, 0, n);
}
out("end split file use time is " + ((System.currentTimeMillis() - stime) / 1000) + "s");
_in.close();
_out.close();
pos = 0;
return newFile;
}
private void put(String data) {
try {
if (fos == null) {
fos = new FileOutputStream(targetFile);
}
fos.write(data.trim().getBytes());
fos.write("\n".getBytes());
} catch (Exception e) {
e.printStackTrace();
}
}
private boolean checkKeywords(String line) {
String[] keys = this.keywords.split(";");
boolean res = false;
for (String key : keys) {
if (line.contains(key)) {
// out(key + " : " + line);
res = true;
break;
}
}
return res;
}
private void checkTargetFile() {
if (this.targetFile == null) {
this.targetFile = new File(getFilePath() + getFileName() + "_" + index + "." + getFileExt());
}
out("out put new file is " + this.targetFile.getAbsolutePath());
}
private String getFileExt() {
String s = file.getName();
return s.substring(s.indexOf(".") + 1);
}
private String getFileName() {
String s = file.getName();
return s.substring(0, s.indexOf(".") - 1);
}
private String getFilePath() {
return file.getAbsolutePath();
}
private void out(String s) {
if (debug) {
if (s != null) {
if (!Thread.interrupted()) {
s = nowDateTime() + " - " + Thread.currentThread().getName() + " " + s;
System.out.println(s);
try {
logQueue.put(s);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
}
private String nowDateTime() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
return sdf.format(new Date());
}
}
interface WorkerListener {
void process(long flen, long process, long pos);
void splitFile(long fileLen, long splitLen, long len);
boolean stop();
}
测试：
日志文件是400MB，创建了5个线程，每个线程处理80MB的数据，耗时大约96~97秒之间，如下图：
分割出来的日志文件：
下面是System.out.println打印出来的日志：
再测试一次，下面是用单线程去处理400MB的数据：
测试完毕的日志数据，需要耗时333秒，如果使用线程，要快上3倍多：
