Java创建100MB压缩的csv文件性能问题

Question

我需要在5秒内创建100mb的压缩文件，其中包含使用java的CSV文件。我已经创建了包含CSV文件的test.zip，但它花费了太多的时间（〜30秒）来生成zip文件。这里是我到目前为止已经编写的代码：

ByteArrayOutputStream baos = new ByteArrayOutputStream(); 
/* Create instance of ZipOutputStream to create ZIP file. */ 
ZipOutputStream zipOutputStream = new ZipOutputStream(baos); 

/* Create ZIP entry for file.The file which is created put into the 
* zip file.File is not on the disk, csvFileName indicates only the 
* file name to be put into the zip 
*/ 
ZipEntry zipEntry = new ZipEntry("Test.zip"); 

zipOutputStream.putNextEntry(zipEntry); 

/* Create OutputStreamWriter for CSV. There is no need for staging 
* the CSV on filesystem . Directly write bytes to the output stream. 
*/ 
BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(zipOutputStream, "UTF-8")); 

CsvListWriter csvListWriter = new CsvListWriter(bufferedWriter, CsvPreference.EXCEL_PREFERENCE); 

/* Write the CSV header to the generated CSV file. */ 
csvListWriter.writeHeader(CSVGeneratorConstant.CSV_HEADERS); 

/* Logic to Write the content to CSV */ 
long startTime = System.currentTimeMillis(); 

for (int rowIdx = 0; rowIdx < 7000000; rowIdx++) { 
    final List<String> rowContent = new LinkedList<String>(); 
    for (int colIdx = 0; colIdx < 6; colIdx++) { 
     String str = "R" + rowIdx + "C" + colIdx + " FieldContent"; 
     rowContent.add(str); 
    } 
    csvListWriter.write(rowContent); 
} 
long stopTime = System.currentTimeMillis(); 
long elapsedTime = stopTime - startTime; 
System.out.println("time==" + elapsedTime/1000f + "Seconds"); 

System.out.println("Size=====" + baos.size()/(Math.pow(1024, 2)) + "MB"); 

csvListWriter.close(); 
bufferedWriter.close(); 
zipOutputStream.close(); 
baos.close(); 

我现在用的是超级CSV库，但我也试图在内存中创建的压缩文件，而不无功而返超级CSV库。你能帮我么？

Answer 1

您的测试数据大约为1GB，压缩到100MB。根据您的硬件，可能无法实现5秒钟的性能。

我把一个快速和肮脏的基准放在一起，突出了写入到zip文件的性能影响。

• 写入CSV与String.join()：9.6s
• 写入CSV具有超强CSV：12.7s
• 写入CSV拉链内的String.join()：18.6s
• 写入CSV具有超强CSV拉链内：22.5s

看起来，使用Super CSV（〜122％）有一点点开销，但只需写入一个zip文件几乎会使时间增加一倍（〜190％），而不管是否Supe使用CSV。

下面是4个场景的代码。与您提供的代码不同，我直接写入一个文件（我没有注意到写入磁盘与写入内存之间的任何差异，即ByteArrayOutputStream）。我也跳过了关于超级CSV示例的BufferedWriter，因为它已经在内部使用了它，并且我使用了试用资源来使事情更加清洁。

@Test 
public void testWriteToCsvFileWithSuperCSV() throws Exception { 
    long startTime = System.currentTimeMillis(); 

    try (FileOutputStream csvFile = new FileOutputStream(new File("supercsv.csv")); 
     ICsvListWriter writer = new CsvListWriter(new OutputStreamWriter(csvFile, "UTF-8"), CsvPreference.EXCEL_PREFERENCE) 
    ){ 
     for (int rowIdx = 0; rowIdx < 7000000; rowIdx++) { 
      final List<String> rowContent = new LinkedList<>(); 
      for (int colIdx = 0; colIdx < 6; colIdx++) { 
       String str = "R" + rowIdx + "C" + colIdx + " FieldContent"; 
       rowContent.add(str); 
      } 
      writer.write(rowContent); 
     } 
    } 

    long stopTime = System.currentTimeMillis(); 
    long elapsedTime = stopTime - startTime; 
    System.out.println("Writing to CSV with Super CSV took " + (elapsedTime/1000f) + " seconds"); 
} 

@Test 
public void testWriteToCsvFileWithinZipWithSuperCSV() throws Exception { 
    long startTime = System.currentTimeMillis(); 

    try (FileOutputStream zipFile = new FileOutputStream(new File("supercsv.zip")); 
     ZipOutputStream zos = new ZipOutputStream(zipFile); 
     ICsvListWriter writer = new CsvListWriter(new OutputStreamWriter(zos, "UTF-8"), CsvPreference.EXCEL_PREFERENCE) 
    ){ 

     ZipEntry csvFile = new ZipEntry("supercsvwithinzip.csv"); 
     zos.putNextEntry(csvFile); 

     for (int rowIdx = 0; rowIdx < 7000000; rowIdx++) { 
      final List<String> rowContent = new LinkedList<>(); 
      for (int colIdx = 0; colIdx < 6; colIdx++) { 
       String str = "R" + rowIdx + "C" + colIdx + " FieldContent"; 
       rowContent.add(str); 
      } 
      writer.write(rowContent); 
     } 
    } 

    long stopTime = System.currentTimeMillis(); 
    long elapsedTime = stopTime - startTime; 
    System.out.println("Writing to CSV within zip file with Super CSV took " + (elapsedTime/1000f) + " seconds"); 
} 

@Test 
public void testWriteToCsvFileWithStringJoin() throws Exception { 
    long startTime = System.currentTimeMillis(); 

    try (FileOutputStream textFile = new FileOutputStream(new File("join.csv")); 
     BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(textFile, "UTF-8")); 
    ){ 

     for (int rowIdx = 0; rowIdx < 7000000; rowIdx++) { 
      final List<String> rowContent = new LinkedList<>(); 
      for (int colIdx = 0; colIdx < 6; colIdx++) { 
       String str = "R" + rowIdx + "C" + colIdx + " FieldContent"; 
       rowContent.add(str); 
      } 
      writer.append(String.join(",", rowContent) + "\n"); 
     } 
    } 

    long stopTime = System.currentTimeMillis(); 
    long elapsedTime = stopTime - startTime; 
    System.out.println("Writing to CSV with String.join() took " + (elapsedTime/1000f) + " seconds"); 
} 

@Test 
public void testWriteToCsvFileWithinZipWithStringJoin() throws Exception { 
    long startTime = System.currentTimeMillis(); 

    try (FileOutputStream zipFile = new FileOutputStream(new File("join.zip")); 
     ZipOutputStream zos = new ZipOutputStream(zipFile); 
     BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(zos, "UTF-8")); 
    ){ 

     ZipEntry csvFile = new ZipEntry("joinwithinzip.csv"); 
     zos.putNextEntry(csvFile); 

     for (int rowIdx = 0; rowIdx < 7000000; rowIdx++) { 
      final List<String> rowContent = new LinkedList<>(); 
      for (int colIdx = 0; colIdx < 6; colIdx++) { 
       String str = "R" + rowIdx + "C" + colIdx + " FieldContent"; 
       rowContent.add(str); 
      } 
      writer.append(String.join(",", rowContent) + "\n"); 
     } 
    } 

    long stopTime = System.currentTimeMillis(); 
    long elapsedTime = stopTime - startTime; 
    System.out.println("Writing to CSV within zip with String.join() took " + (elapsedTime/1000f) + " seconds"); 
}