1
我有一个要求来聚合来自几个不同的网站（主要是HTML页面和PDF文档）的内容。我目前正在试验Heritrix（3.2.0）以查看它是否能满足我的需求。Heritrix内容过滤
尽管文档非常详细，但引擎似乎并没有像我期望的那样工作。我已经设置了一些简单的工作，并配置了很多不同的方式，但不管我做了什么，我发现Heritrix要么拉下太多的内容，要么什么都不拉。
下面是我想要做的一个例子。我正在将Heritrix指向URL，如... example.com/news/speeches。这是一个带有HTML表格的网页，其中包含指向个人演讲（ex..example.com/news/speech/speech1.html，xample.com/news/speech/speech2.html等）的链接。我真的只需要在父页面下一级的HTML和PDF文档。我希望阻止Heritrix进行比1级更深的导航，如果不低于example.com域上的此特定路径，阻止Heritrix拉动内容，阻止它导航到另一个域，并将其限制为html和pdf内容。
以下配置是什么，我觉得应该工作，但不
<bean id="longerOverrides" class="org.springframework.beans.factory.config.PropertyOverrideConfigurer">
<property name="properties">
<props>
<prop key="seeds.textSource.value">
# URLS HERE
example.com/news/speeches
</prop>
</props>
</property>
</bean>
<bean id="scope" class="org.archive.modules.deciderules.DecideRuleSequence">
<!-- <property name="logToFile" value="false" /> -->
<property name="rules">
<list>
<!-- Begin by REJECTing all... -->
<bean class="org.archive.modules.deciderules.RejectDecideRule">
</bean>
<!-- ...then ACCEPT those within configured/seed-implied SURT prefixes... -->
<bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule">
<!-- <property name="seedsAsSurtPrefixes" value="true" /> -->
<!-- <property name="alsoCheckVia" value="false" /> -->
<!-- <property name="surtsSourceFile" value="" /> -->
<!-- <property name="surtsDumpFile" value="${launchId}/surts.dump" /> -->
<property name="surtsSource">
<bean class="org.archive.spring.ConfigString">
<property name="value">
<value>
example.com/news/speeches
</value>
</property>
</bean>
</property>
</bean>
<!-- ...and REJECT those from a configurable (initially empty) set of URI regexes... -->
<bean class="org.archive.modules.deciderules.MatchesListRegexDecideRule">
<property name="decision" value="REJECT"/>
<property name="listLogicalOr" value="true" />
<property name="regexList">
<list>
<value>.*(?i)(\.(avi|wmv|mpe?g|mp3))$</value>
<value>.*(?i)(\.(rar|zip|tar|gz))$</value>
<value>.*(?i)(\.(xls|odt))$</value>
<value>.*(?i)(\.(xml))$</value>
<value>.*(?i)(\.(txt|conf|pdf))$</value>
<value>.*(?i)(\.(swf))$</value>
<value>.*(?i)(\.(js|css))$</value>
<value>.*(?i)(\.(bmp|gif|jpe?g|png|svg|tiff?))$</value>
</list>
</property>
</bean>
<!-- ...but REJECT those more than a configured link-hop-count from start... -->
<bean class="org.archive.modules.deciderules.TooManyHopsDecideRule">
<!-- <property name="maxHops" value="20" /> -->
</bean>
<!-- ...but ACCEPT those more than a configured link-hop-count from start... -->
<!--bean class="org.archive.modules.deciderules.TransclusionDecideRule"-->
<!-- <property name="maxTransHops" value="2" /> -->
<!-- <property name="maxSpeculativeHops" value="1" /> -->
<!--/bean-->
<!-- ...but REJECT those from a configurable (initially empty) set of REJECT SURTs... -->
<bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule">
<property name="decision" value="REJECT"/>
<property name="seedsAsSurtPrefixes" value="false"/>
<property name="surtsDumpFile" value="${launchId}/negative-surts.dump" />
<!-- <property name="surtsSource">
<bean class="org.archive.spring.ConfigFile">
<property name="path" value="negative-surts.txt" />
</bean>
</property> -->
</bean>
<!-- ...and REJECT those from a configurable (initially empty) set of URI regexes... -->
<bean class="org.archive.modules.deciderules.MatchesListRegexDecideRule">
<property name="decision" value="REJECT"/>
<!-- <property name="listLogicalOr" value="true" /> -->
<!-- <property name="regexList">
<list>
</list>
</property> -->
</bean>
<!-- ...and REJECT those with suspicious repeating path-segments... -->
<bean class="org.archive.modules.deciderules.PathologicalPathDecideRule">
<!-- <property name="maxRepetitions" value="2" /> -->
</bean>
<!-- ...and REJECT those with more than threshold number of path-segments... -->
<bean class="org.archive.modules.deciderules.TooManyPathSegmentsDecideRule">
<!-- <property name="maxPathDepth" value="20" /> -->
</bean>
<!-- ...but always ACCEPT those marked as prerequisitee for another URI... -->
<bean class="org.archive.modules.deciderules.PrerequisiteAcceptDecideRule">
</bean>
<!-- ...but always REJECT those with unsupported URI schemes -->
<bean class="org.archive.modules.deciderules.SchemeNotInSetDecideRule">
</bean>
</list>
</property>
</bean>
我希望我的检索只拉下十几HTML文档作为是所有的包含/语音通道内。大约半小时后，我停止了抓取，因为它正在下载800多个文档，因为我发现它正在向后移动到父级路径。我也尝试了RegEx规则，但没有运气。任何帮助，将不胜感激。