在Apache POI中读取10 MB文件

Question

我正在进行的项目正在尝试读取一个非常大的Excel文件（几百列和大约3000行）并识别一系列字母中的模式。它在小文件上工作得很好，但是当我尝试使用这个文件运行它时，即使我只试图分析前几行，我也会收到java.lang.OutOfMemoryError: Java heap space错误。错误似乎是在Workbook wb = WorkbookFactory.create(new File(filepath));

我已经试过了几本网站上的解决方案，但在任何成功都不会来。我的代码如下：

import java.awt.List; 
import java.io.File; 
import java.io.FileInputStream; 
import java.io.IOException; 
import java.util.ArrayList; 

import org.apache.poi.EncryptedDocumentException; 
import org.apache.poi.openxml4j.exceptions.InvalidFormatException; 
import org.apache.poi.ss.usermodel.Workbook; 
import org.apache.poi.ss.usermodel.WorkbookFactory; 
import org.apache.poi.xssf.usermodel.XSSFCell; 
import org.apache.poi.xssf.usermodel.XSSFRow; 
import org.apache.poi.xssf.usermodel.XSSFSheet; 
import org.apache.poi.xssf.usermodel.XSSFWorkbook; 

public class ExcelReader { 

    public int Reader(File file) throws IOException, EncryptedDocumentException, InvalidFormatException { 
     String filepath = file.getPath(); 
     Workbook wb = WorkbookFactory.create(new File(filepath)); 
     XSSFSheet sheet = (XSSFSheet) wb.getSheetAt(0); 
     XSSFRow row; 
     XSSFCell cell; 
     ArrayList<Integer> list = new ArrayList<Integer>(); 

     int rows; 
     int cols = 0; 
     int temp = 0; 
     rows = sheet.getPhysicalNumberOfRows(); 

     for (int i = 0; i <= 1; i++) { 
      row = sheet.getRow(i); 
      if (row != null) { 
       temp = sheet.getRow(i).getPhysicalNumberOfCells(); 
       if (temp > cols) 
        cols = temp; 
      } 
     } 
     for (int r = 0; r <= 60; r++) { 
      row = sheet.getRow(r); 
      if (row != null) { 
       for (int c = 0; c <= cols; c++) { 
        int numblanks = 0; 
        cell = row.getCell((short) c); 
        if (cell != null) { 
         //System.out.print(cell + "\t\t"); 
        } else { 
         //System.out.print("\t\t"); 
        } 
        if (cell != null && cell.getCellType() == XSSFCell.CELL_TYPE_STRING) { 
         if ("N".equals(cell.getStringCellValue())) { 
          for (int k = c; k <= cols; k++) { 
           if ("-".equals(row.getCell(k).getStringCellValue())) { 
            numblanks++; 
            continue; 
           } 
           if ("S".equals(row.getCell(c + 2 + numblanks).getStringCellValue()) 
             || "T".equals(row.getCell(c + 2 + numblanks).getStringCellValue())) { 
            list.add((int) sheet.getRow(1).getCell(c).getNumericCellValue()); 
            break; 
           } 
          } 
         } 
        } 
       } 
       System.out.println(); 
      } 
     } 
     System.out.println(); 
     System.out.println("Rows: " + rows); 
     System.out.println("Columns: " + cols); 
     System.out.println(list); 
     return temp; 
    } 
} 

谢谢任何​​帮助，你可以给我！

Answer 1

我以前解决过这个问题。我的情况是读取一个包含23万行的23M Excel文件。

增加最大堆大小不是一个好的解决方案。 Apache poi没有流模式来读取数据。这种非流模式会花费太多内存。

我的解决办法是将数据转换为XML，然后使用XMLReader的解析数据。

请检查下面的示例代码：

protected List<Entity> parseData(InputStream in) throws Exception { 
     OPCPackage pkg = OPCPackage.open(in); 
     XSSFReader r = new XSSFReader(pkg); 
     SharedStringsTable sst = r.getSharedStringsTable(); 
     XMLReader parser = fetchSheetParser(sst); 
     XSSFReader.SheetIterator sheets = (XSSFReader.SheetIterator) r.getSheetsData(); 

     while (sheets.hasNext()) { 
      InputStream sheet = sheets.next(); 
      InputSource sheetSource = new InputSource(sheet); 
      parser.parse(sheetSource); 
      sheet.close(); 
      break; // if only need to process one sheet. 
     } 
     return SheetHandler.getRawData(); 
    } 

    private XMLReader fetchSheetParser(SharedStringsTable sst) throws SAXException { 
     XMLReader parser = 
       XMLReaderFactory.createXMLReader(); 
     ContentHandler handler = new SheetHandler(sst); 
     parser.setContentHandler(handler); 
     return parser; 
    } 

    private static class SheetHandler extends DefaultHandler { 

     private SharedStringsTable sst; 
     private String lastContents; 
     private boolean nextIsString; 
     private boolean nextIsInlineString; 
     private boolean nextIsNull; 

     private SheetHandler(SharedStringsTable sst) { 
      this.sst = sst; 
      rawData = new ArrayList<Entity>(); 
     } 

     public static List<Entity> getRawData() { 
      return rawData; 
     } 


     @Override 
     public void startElement(String uri, String localName, String name, 
           Attributes attributes) throws SAXException { 

     } 

     @Override 
     public void endElement(String uri, String localName, String name) 
       throws SAXException { 


     } 

     @Override 
     public void characters(char[] ch, int start, int length) 
       throws SAXException { 
      lastContents += new String(ch, start, length); 
     } 
    } 
}