Boa tarde pessoal. Estou tendo um problema com requests para raspagem de dados.
A função principal é a que segue:
def raspa_dados(lista_de_links, ministerio):
links = []
autores = []
chamadas = []
datas = []
ementas = []
for i in lista_de_links:
time.sleep(0.01)
html = trata_html(requests.get(i).text)
soup = BeautifulSoup(html, 'lxml')
orgao = soup.findAll('span', class_="orgao-dou-data")[0].get_text()
chamada = soup.findAll('p', class_="identifica")[0].get_text()
data = soup.findAll('span', class_="publicado-dou-data")[0].get_text()
ementa = soup.findAll('p', class_="ementa")
if ministerio in orgao:
if chamada.split()[0] in tipos or chamada.split()[0] in primeira_maiuscula(tipos):
autores.append(orgao)
chamadas.append(chamada)
datas.append(data)
links.append(i)
if len(ementa) >= 1:
ementas.append(ementa[0].get_text())
else:
ementas.append(None)
return links, autores, chamadas, datas, ementas
Recentemente eu adicionei esta linha do time.sleep(0.01)
na esperança de solucionar o problema.
Sigo recebendo a seguinte mensagem de erro:
ConnectionResetError Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/urllib3/response.py in _error_catcher(self)
361 try:
--> 362 yield
363
18 frames
ConnectionResetError: [Errno 104] Connection reset by peer
During handling of the above exception, another exception occurred:
ProtocolError Traceback (most recent call last)
ProtocolError: ("Connection broken: ConnectionResetError(104, 'Connection reset by peer')", ConnectionResetError(104, 'Connection reset by peer'))
During handling of the above exception, another exception occurred:
ChunkedEncodingError Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/requests/models.py in generate()
752 yield chunk
753 except ProtocolError as e:
--> 754 raise ChunkedEncodingError(e)
755 except DecodeError as e:
756 raise ContentDecodingError(e)
ChunkedEncodingError: ("Connection broken: ConnectionResetError(104, 'Connection reset by peer')", ConnectionResetError(104, 'Connection reset by peer'))
Alguém sabe me dizer o que está errado?