LoginSignup
0
0

More than 5 years have passed since last update.

@souka-90

log大文件中高效率检索固定字符串之1

Last updated at Posted at 2018-02-21

1、使用方式之一、先把文件读到内存中，使用java中的正则表达式去匹配，使用的是Matcher Pattern等类
2、使用方式之二、先把文件读到内存中，在内存中检索文件所有换行符，并根据换行符和字substring（）截取每一行文字进行判断。不适用正则表达式，使用的是indexOf()、substring()两个函数。

【总结】
方式1 逻辑简单，只需要准备好对应的正则表达式就可以.但执行效率非常慢，非常耗时。1380行数据，查找其中3行数据，需要6s左右
方式2 实现逻辑需要自己写实现代码（indexOf()\substring()），具体代码如下 getMatchrfLines（）方法

package com.log;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.sql.main.CommonUtil;

public class LoggerFileDiv {
    private static final String LOG_FILE_PATH ="C:\\Users\\cloud-user\\work\\テスト\\ITテスト2\\xml\\";
    private static String applogcnts="";
    private static String errlogcnts="";
    private static int[] nCountArry;
    private static int[] nCountArryErr;

    static{
        initFileCnts("MRO030");

        int r =0;
        nCountArry=new int[12000];//\n index array
        int count=0;
        while((r=applogcnts.indexOf("\n",r+50)) > 0){
            nCountArry[count]=r;
            count++;
        }
        //err log
        r =0;
        count=0;
        nCountArryErr=new int[12000];//\n index array
        while((r=errlogcnts.indexOf("\n",r+20)) > 0){
            nCountArryErr[count]=r;
            count++;
        }
    }


    public static void main(String[] args) {
//      HashMap data =new HashMap();
//      data.put("null", null);
//      data.put("null2", null);
//      data.put(null, "null2222");
//      System.out.println(data.get(null));;
//      
//      System.out.println(data);

//      String pattern =",20180220113652,A004002990501,029905010001167,";
                //,20180220132637,A004002990501,029905010001340,
//      divFileByTranatmseq("MRO020");
        divFileByTranatmseq("MRO030");
//      copyRequestXml("MRO020");
//      copyRequestXml("MRO030");
    }
    public static void copyRequestXml(String progrmid){
        File dirP =new File(LOG_FILE_PATH +progrmid +"\\output\\day_test\\");
        String inputXmlPath =LOG_FILE_PATH +"\\input\\";

        List<String> lst =new ArrayList<String>();
        lst.add("123458");
        lst.add("acbsdsd");
        lst.add("212gffff");
//      lst.forEach((n) -> {System.out.println(n);});
        //lambda 公式
        dirP.listFiles( 
                ( dir , name) ->{  
                    System.out.println(name);
                    return true;
                }

                );


//      dirP.listFiles( (File dir, String name) -> {
//          if(!CommonUtil.isEmpty(dir) && name.endsWith("-request.xml")){
//              System.out.println("logggg:"+name);
//              return true;
//          }else {
//              return false;
//          }
//      }) ;


//      for(File caseDir : dir.listFiles(new MROPrefixFilter())){
//          for(File xml : caseDir.listFiles(new RequestXmlFilenameFilter())){
//              xml.renameTo(new File(inputXmlPath +xml.getName()));
//          }
//      }

    }


    public static String getTranatmseqFromXml(String progrmid,String xmlName){
        String[] tags =new String[]{"<transaction_date>(.*)</transaction_date>",
                    "<terminal_no>(.*)</terminal_no>",
                "<terminal_serial_no>(.*)</terminal_serial_no>"};

        String xmlPath =LOG_FILE_PATH+progrmid+"\\output\\"+xmlName;
        File dir =new File(xmlPath);
        BufferedReader bfr = null;  
        StringBuffer xmlStr=new StringBuffer();
        StringBuffer sb =new StringBuffer();
        try {
            bfr=new BufferedReader(new InputStreamReader(new FileInputStream(dir), "Shift_JIS"));
            while(bfr.ready()){
                xmlStr.append(bfr.readLine()+"\r\n");
            }
            for(int i=0;i<3;i++){
                Pattern p =Pattern.compile(tags[i]);
                Matcher m =p.matcher(xmlStr);
                if(m.find()){
                    sb.append(","+m.group(1));
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            try {
                bfr.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return sb.toString();

    }

    public static void divFileByTranatmseq(String progrmid){
        File dir =new File(LOG_FILE_PATH+progrmid+"\\output\\");
        int i=0;
        if(dir.isDirectory()){
            File[] caseDirs =dir.listFiles(new MROPrefixFilter());
            for (File caseDir : caseDirs){
                i++;
                String caseNo =caseDir.getName();
                String responseXml =caseNo+"\\"+caseNo+"-request.xml";
                String patternStr =getTranatmseqFromXml(progrmid,responseXml);
                //getProperLogcnts(progrmid,caseNo,patternStr);
                getMatchrfLines(progrmid,caseNo,patternStr);
//              if(i>=20) break;
            }
        }

    }

    public static void getProperLogcnts(String progrmid,String caseno,String tranatmseq){
        //,20180220132637,A004002990501,029905010001340,
        String pattern="(.*"+tranatmseq+".*)";
        File dir =new File(LOG_FILE_PATH+progrmid+"\\");

        if(dir.isDirectory()){
            File[] loggFs=dir.listFiles((dir2,name) ->{
                if(name.endsWith(".log")) return true;
                else return false;});

            //read file line by line
            BufferedReader bfr =null;

            for(File f : loggFs){
                String fname =f.getName();
                StringBuffer findedLineCnts =new StringBuffer();
                String preHeadLine ="";
                try {
                     bfr=new BufferedReader(new InputStreamReader(new FileInputStream(f), "UTF-8"));

                     boolean headLineAddedFlg =false;
                     boolean flushFileFlg =false;
                     while(bfr.ready()){

                         String line =bfr.readLine();

                         if(matchLine(line,pattern)){
                             if(!headLineAddedFlg && fname.contains("mr-on-ap")){
                                 findedLineCnts.append(preHeadLine+"\r\n");
                                 headLineAddedFlg=true;
                             }
                             findedLineCnts.append(line+"\r\n");
                             flushFileFlg=true;
                         }else {
                             if(flushFileFlg){
                                 //write file to path
                                 String logfilepath =LOG_FILE_PATH+progrmid+"\\output\\"+caseno+"\\"+fname;
                                 CommonUtil.writeString(logfilepath, findedLineCnts.toString());
                                 System.out.println(caseno+" log writen.");

                                 break;
                             }
                             preHeadLine=line;//pre line 

                             headLineAddedFlg =false;
                             flushFileFlg=false;

                        }
//                       fileCnts.append();
                     }
                     bfr.close();

                } catch (Exception e) {
                    e.printStackTrace();
                    break;
                }
            }
        }
    }

    public static void initFileCnts(String progrmid){
        File aplogFile =new File(LOG_FILE_PATH+progrmid+"\\mr-on-ap.log");
        File errlogFile =new File(LOG_FILE_PATH+progrmid+"\\mr-on-err.log");
        BufferedReader bfr =null;
        BufferedReader bfr2 =null;
        StringBuffer applogsb =new StringBuffer();
        StringBuffer errlogsb =new StringBuffer();
         try {
            bfr=new BufferedReader(new InputStreamReader(new FileInputStream(aplogFile), "UTF-8"));
            bfr2=new BufferedReader(new InputStreamReader(new FileInputStream(errlogFile), "UTF-8"));
            while(bfr.ready()){
                applogsb.append(bfr.readLine()+"\r\n"); 
             }
            while(bfr2.ready()){
                errlogsb.append(bfr2.readLine()+"\r\n"); 
             }
            applogcnts = applogsb.toString();
            errlogcnts = errlogsb.toString();

            System.out.println("applogcnts size:"+applogcnts.length());
            System.out.println("errlogcnts size:"+errlogcnts.length());

        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            try {
                bfr.close();
                bfr2.close();
            } catch (IOException e) {
                e.printStackTrace();
            }

        }


    }
    private static boolean matchLine(String cnts,String pattern ){
        Pattern p =Pattern.compile(pattern);
        Matcher m =p.matcher(cnts);
        return m.find();
    }
    private static void  getMatchrfLines(String progrmid,String caseno,String tranatmseq){
        String test =",20180220113652,A004002990501,029905010001166";
        String pattern="(.*"+test+".*)";
        Pattern p =Pattern.compile(pattern);
        Matcher m =p.matcher(applogcnts);

        long starttime =System.currentTimeMillis();

//      while(m.find()){
//          System.out.println(m.group(1));
//      }
//      int r =0;
//      int[] nCountArry=new int[1500];//\n index array
//      int count=0;
//      while((r=applogcnts.indexOf("\n",r+50)) > 0){
////            System.out.println("index:"+r);
//          nCountArry[count]=r;
//          count++;
//      }
        int startpos =0;
        StringBuffer sbCnts = new StringBuffer();
        boolean headAddedFlg =false;
        String prelineStr ="";

        //applog file 
        for(int i=0;i<nCountArry.length;i++){
            if(nCountArry[i] <=0) break;
            String lineStr = applogcnts.substring(startpos, nCountArry[i]);

            if(lineStr.indexOf(tranatmseq)>0){
                if(!headAddedFlg){
                    sbCnts.append(prelineStr+"\n");
                    headAddedFlg=true;
                }
                sbCnts.append(lineStr+"\n");
            }
            prelineStr=lineStr;
            startpos = nCountArry[i] +1;
        }

        String applogfilepath =LOG_FILE_PATH+progrmid+"\\output\\"+caseno+"\\"+"mr-on-ap.log";
        CommonUtil.writeString(applogfilepath, sbCnts.toString());

        startpos =0;
        sbCnts = new StringBuffer();
        prelineStr ="";

        //errlog file 
        for(int i=0;i<nCountArryErr.length;i++){
            if(nCountArryErr[i] <=0) break;
            String lineStr = errlogcnts.substring(startpos, nCountArryErr[i]);

            if(lineStr.indexOf(tranatmseq)>0){
                sbCnts.append(lineStr+"\n");
            }
            prelineStr=lineStr;
            startpos = nCountArryErr[i] +1;
        }
        String errlogfilepath =LOG_FILE_PATH+progrmid+"\\output\\"+caseno+"\\"+"mr-on-err.log";
        CommonUtil.writeString(errlogfilepath, sbCnts.toString());

        long endtime =System.currentTimeMillis();
        System.out.println(caseno+"execute time :"+(endtime-starttime)+"ms");
    }

}
class LogFileFilenameFilter implements FilenameFilter{

    @Override
    public boolean accept(File dir, String name) {

        if(!CommonUtil.isEmpty(name) && name.endsWith(".log")){
            System.out.println("logggg:"+name);
            return true;
        }else {
            return false;
        }
    }

}
class RequestXmlFilenameFilter implements FilenameFilter{

    @Override
    public boolean accept(File dir, String name) {

        if(!CommonUtil.isEmpty(name) && name.endsWith("-request.xml")){
            System.out.println("logggg:"+name);
            return true;
        }else {
            return false;
        }
    }

}
class MROPrefixFilter implements FilenameFilter{

    @Override
    public boolean accept(File dir, String name) {

        if(!CommonUtil.isEmpty(name) && name.startsWith("MRO")){
//          System.out.println("MRO:"+name);
            return true;
        }else {
            return false;
        }
    }

}
0
0
0

Register as a new user and use Qiita more conveniently

  1. You get articles that match your needs
  2. You can efficiently read back useful information
  3. You can use dark theme
What you can do with signing up
Sign upLogin
0
0