update.py

import os
import re
import shutil
from urllib.request import urlopen, Request

base_url = "https://www.sintef.no"
index_path = "/projectweb/top/vrptw/homberger-benchmark/%s-customers/"
num_customers_list = ["200", "400", "600", "800", "1000"]
pattern = re.compile('"(/contentassets/.*?/((c|r|rc)\d+_\d+_\d+).*?\.txt)"')
headers = {"User-Agent": ""}

for num_customers in num_customers_list:
    url = base_url + index_path % num_customers
    with urlopen(Request(url, headers=headers), timeout=10) as response:
        body = response.read().decode()
    for path, instance_name, _ in pattern.findall(body):
        print(path)
        file_path = os.path.join(num_customers, instance_name + ".sol")
        with urlopen(Request(base_url + path, headers=headers), timeout=10) as response:
            with open(file_path, "wb") as file:
                shutil.copyfileobj(response, file)