html文件的lucene索引

Question

尊敬的用户我正在使用apache lucene进行索引和搜索。 我必须索引存储在本地计算机光盘上的html文件。我必须对html文件的文件名和内容进行索引。我能够将文件名存储在lucene索引中，但不能存储html文件内容，该文件内容不仅应该索引数据，还会索引整个页面组成的图像链接和url，以及如何从索引文件 访问内容以索引我使用下面的代码：

File indexDir = new File(indexpath); 
    File dataDir = new File(datapath); 
    String suffix = ".htm"; 
    IndexWriter indexWriter = new IndexWriter(
      FSDirectory.open(indexDir), 
      new SimpleAnalyzer(), 
      true, 
      IndexWriter.MaxFieldLength.LIMITED); 
    indexWriter.setUseCompoundFile(false); 
    indexDirectory(indexWriter, dataDir, suffix); 

    numIndexed = indexWriter.maxDoc(); 
    indexWriter.optimize(); 
    indexWriter.close(); 


private void indexDirectory(IndexWriter indexWriter, File dataDir, String suffix) throws IOException { 
    try { 
     for (File f : dataDir.listFiles()) { 
      if (f.isDirectory()) { 
       indexDirectory(indexWriter, f, suffix); 
      } else { 
       indexFileWithIndexWriter(indexWriter, f, suffix); 
      } 
     } 
    } catch (Exception ex) { 
     System.out.println("exception 2 is" + ex); 
    } 
} 

private void indexFileWithIndexWriter(IndexWriter indexWriter, File f, 
    String suffix) throws IOException { 
    try { 
     if (f.isHidden() || f.isDirectory() || !f.canRead() || !f.exists()) { 
      return; 
     } 
     if (suffix != null && !f.getName().endsWith(suffix)) { 
      return; 
     } 
     Document doc = new Document(); 
     doc.add(new Field("contents", new FileReader(f))); 
     doc.add(new Field("filename", f.getFileName(), 
       Field.Store.YES, Field.Index.ANALYZED)); 
     indexWriter.addDocument(doc); 
    } catch (Exception ex) { 
     System.out.println("exception 4 is" + ex); 
    } 
} 

在此先感谢

Answer 1

这行代码是为什么不被存储你的内容的原因：

doc.add(new Field("contents", new FileReader(f))); 

这种方法不存储内容被索引。

如果您尝试索引HTML文件，请尝试使用JTidy。这将使这个过程变得更容易。

示例代码：

public class JTidyHTMLHandler { 

    public org.apache.lucene.document.Document getDocument(InputStream is) throws DocumentHandlerException { 
     Tidy tidy = new Tidy(); 
     tidy.setQuiet(true); 
     tidy.setShowWarnings(false); 
     org.w3c.dom.Document root = tidy.parseDOM(is, null); 
     Element rawDoc = root.getDocumentElement(); 

     org.apache.lucene.document.Document doc = 
       new org.apache.lucene.document.Document(); 

     String body = getBody(rawDoc); 

     if ((body != null) && (!body.equals(""))) { 
      doc.add(new Field("contents", body, Field.Store.NO, Field.Index.ANALYZED)); 
     } 

     return doc; 
    } 

    protected String getTitle(Element rawDoc) { 
     if (rawDoc == null) { 
      return null; 
     } 

     String title = ""; 

     NodeList children = rawDoc.getElementsByTagName("title"); 
     if (children.getLength() > 0) { 
      Element titleElement = ((Element) children.item(0)); 
      Text text = (Text) titleElement.getFirstChild(); 
      if (text != null) { 
       title = text.getData(); 
      } 
     } 
     return title; 
    } 

    protected String getBody(Element rawDoc) { 
     if (rawDoc == null) { 
      return null; 
     } 

     String body = ""; 
     NodeList children = rawDoc.getElementsByTagName("body"); 
     if (children.getLength() > 0) { 
      body = getText(children.item(0)); 
     } 
     return body; 
    } 

    protected String getText(Node node) { 
     NodeList children = node.getChildNodes(); 
     StringBuffer sb = new StringBuffer(); 
     for (int i = 0; i < children.getLength(); i++) { 
      Node child = children.item(i); 
      switch (child.getNodeType()) { 
       case Node.ELEMENT_NODE: 
        sb.append(getText(child)); 
        sb.append(" "); 
        break; 
       case Node.TEXT_NODE: 
        sb.append(((Text) child).getData()); 
        break; 
      } 
     } 
     return sb.toString(); 
    } 
} 

从一个URL获得的InputStream：

InputStream stream = new FileInputStream(new File (htmlFile));