使用python从网站下载文件

Question

我需要从（https://www.sec.gov/litigation/suspensions.shtml）给定网站下载所有文件。它有从1995年到2017年的数据，每年里面有多个需要下载的文件的链接。 Th文件使用.pdf，.htm和.txt格式。我试图通过查看各种教程来抓取数据，但是我需要做的与通常的网页抓取教程不同。我使用了下面的代码，但它并没有达到我的目的。我是python的新手，我在这里被困在如何前进的道路上。任何人都可以请建议需要做什么。

import requests 
from bs4 import BeautifulSoup 

r = requests.get("https://www.sec.gov/litigation/suspensions.shtml") 
r.content 

soup = BeautifulSoup(r.content) 
print soup.prettify() 

links = soup.find_all("a") 

for link in links: 
     print "<a href= '%s'>%s</a>" %(link.get("href"), link.text) 

g_data = soup.find_all("p", {"id": "archive-links"}) 
print g_data 

for item in g_data: 
    print item.text 

Answer 1

这应该做的工作。检查了Python 3.6，但代码应该是Python2.7兼容的。 主要想法是找到每年的链接，然后抓取每年的pdf，htm和txt文件的所有链接。

from __future__ import print_function 

import requests 
from bs4 import BeautifulSoup 


def file_links_filter(tag): 
    """ 
    Tags filter. Return True for links that ends with 'pdf', 'htm' or 'txt' 
    """ 
    if isinstance(tag, str): 
     return tag.endswith('pdf') or tag.endswith('htm') or tag.endswith('txt') 


def get_links(tags_list): 
    return [WEB_ROOT + tag.attrs['href'] for tag in tags_list] 


def download_file(file_link, folder): 
    file = requests.get(file_link).content 
    name = file_link.split('/')[-1] 
    save_path = folder + name 

    print("Saving file:", save_path) 
    with open(save_path, 'wb') as fp: 
     fp.write(file) 


WEB_ROOT = 'https://www.sec.gov' 
SAVE_FOLDER = '~/download_files/' # directory in which files will be downloaded 

r = requests.get("https://www.sec.gov/litigation/suspensions.shtml") 

soup = BeautifulSoup(r.content, 'html.parser') 

years = soup.select("p#archive-links > a") # css selector for all <a> inside <p id='archive'> tag 
years_links = get_links(years) 

links_to_download = [] 
for year_link in years_links: 
    page = requests.get(year_link) 
    beautiful_page = BeautifulSoup(page.content, 'html.parser') 

    links = beautiful_page.find_all("a", href=file_links_filter) 
    links = get_links(links) 

    links_to_download.extend(links) 

# make set to exclude duplicate links 
links_to_download = set(links_to_download) 

print("Got links:", links_to_download) 

for link in set(links_to_download): 
    download_file(link, SAVE_FOLDER)