Commit 9906a782 authored by Eliot Berriot's avatar Eliot Berriot

Merge branch 'release/0.2.4'

parents 10fe6481 a6c1f0b3
Pipeline #191 passed with stages
in 40 seconds
......@@ -103,10 +103,13 @@ def build_database(**kwargs):
wrapper.xpath("ul/li/a"))
for l in links:
url = ROOT_URL + l.attrib['href']
i = int(l.text.split('-')[-1])
try:
i = int(l.text.split('-')[-1])
except ValueError:
i = int(url.split('-')[-1])
name = l.getparent().text_content().split(' - ')[-1]
id_url_mapping[i] = (url, name)
logger.info('Fetching individual scps'.format(i))
logger.info('Fetching individual scps...')
sem = asyncio.Semaphore(kwargs['concurrency'])
loop = asyncio.get_event_loop()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment