
# %pip install lxml
# %pip install requests


from io import StringIO
from mimetypes import guess_extension
from os import makedirs, path, rmdir
from datetime import date
from lxml import etree
import requests


ns_map = {'xmlns': "http://www.openarchives.org/OAI/2.0/",
          'mets': "http://www.loc.gov/METS/",
          'xlink': "http://www.w3.org/1999/xlink",
          'mods': "http://www.loc.gov/mods/v3"}


newspaper_series_url = 'https://visuallibrary.net/dps/oai/?verb=GetRecord&metadataPrefix=mets&identifier=436884'


# formats = ['Fulltext', 'Min', 'Thumbs', 'Default', 'Max', 'Download']
formats = ['Fulltext', 'Max', 'Download']


base = "downloads"


folder_structure = ['base', 'newspaper_series', 'newspaper', 'year', 'issue', 'fmt']


# parts that will be replaced: NEWSPAPER_SERIES_NAME, NEWSPAPER_NAME, YEAR_NAME, ISSUE_NAME, PAGE_ID, PAGE_NO
page_name_format = 'Seite PAGE_NO'


year_name_format = '%Y'


issue_name_format = '%Y-%m-%d'
issue_name_granularity = True


newspaper_series_name = ""
newspaper_names = {
    'newspaper_id': 'name',
}


# append_VL_ID = ['newspaper_series', 'newspaper', 'issue', 'year', 'page']
append_VL_ID = ['page']


s = requests.Session()
s.headers.update({'User-Agent': 'METS_Downloader'})

# get root node as lxml node from a url
get_root = lambda url: etree.parse(StringIO(s.get(url).text)).getroot()

get_newspapers_from_newspaper_series = lambda newspaper_series_url: \
    get_root(newspaper_series_url).xpath('//mets:structMap//@xlink:href', namespaces=ns_map)

get_years_from_newspaper = lambda newspaper_url: \
    get_root(newspaper_url).xpath('//mets:structMap//@xlink:href', namespaces=ns_map)

get_issues_from_year = lambda year_url : \
    get_root(year_url).xpath('//mets:structMap//*[@TYPE="day"]//@xlink:href', namespaces=ns_map)

def get_pages_from_issue(url, fmt):
    """Get pages from an issue-url, provide format which should be extracted, return list of links to pages."""
    root = get_root(url)
    page_links = root.xpath(f'//mets:fileGrp[@USE="{fmt.upper()}"]//@xlink:href', namespaces=ns_map)
    return page_links

def get_newspaper_series_name(newspaper_series_url, newspaper_series_name, append_VL_ID):
    if newspaper_series_name:
        newspaper_series_name_formatted = newspaper_series_name
    else:
        root = get_root(newspaper_series_url)
        parts = root.xpath(f'//mets:dmdSec[@ID="md{get_id(newspaper_series_url)}"]//mods:titleInfo', namespaces=ns_map)[0]
        newspaper_series_name_formatted = ""
        for part in parts:
            newspaper_series_name_formatted += f'{part.text}. '

        newspaper_series_name_formatted = newspaper_series_name_formatted.rstrip('. ')

    if 'newspaper_series' in append_VL_ID:
        newspaper_series_name_formatted += f'.{get_id(newspaper_series_url)}'

    return newspaper_series_name_formatted

def get_newspaper_name(newspaper_url, newspaper_names, append_VL_ID):
    if get_id(newspaper_url) in newspaper_names:
        newspapper_name_formatted = newspaper_names[get_id(newspaper_url)]
    else:
        root = get_root(newspaper_url)
        parts = root.xpath(f'//mets:dmdSec[@ID="md{get_id(newspaper_url)}"]//mods:titleInfo', namespaces=ns_map)[0]
        newspapper_name_formatted = ""
        for part in parts:
            newspapper_name_formatted += f'{part.text}. '

        newspapper_name_formatted = newspapper_name_formatted.rstrip('. ')

    if 'newspaper' in append_VL_ID:
        newspapper_name_formatted += f'.{get_id(newspaper_url)}'

    return newspapper_name_formatted

def get_year_name(year_url, year_date_format, append_VL_ID):
    root = get_root(year_url)
    year_date_string = root.xpath(f'//mets:dmdSec[@ID="md{get_id(year_url)}"]//mods:date', namespaces=ns_map)[0].text
    year_date = date(int(year_date_string), 1, 1)
    year_name_formatted = year_date.strftime(year_date_format)

    if 'year' in append_VL_ID:
        year_name_formatted += f'.{get_id(year_url)}'

    return year_name_formatted

def get_issue_name(issue_url, issue_date_format, issue_name_granularity, append_VL_ID):
    root = get_root(issue_url)
    issue_date_string = root.xpath(f'//mets:dmdSec[@ID="md{get_id(issue_url)}"]//mods:date', namespaces=ns_map)[0].text
    issue_date = date.fromisoformat(issue_date_string)
    issue_name_formatted = issue_date.strftime(issue_date_format)

    # if issue_name_granularity == True, append detail information like number, title about issue, if provided
    if issue_name_granularity:
        detail = root.xpath(f'//mets:dmdSec[@ID="md{get_id(issue_url)}"]//mods:detail[@type="issue"]', namespaces=ns_map)
        if detail:
            for child in detail[0]:
                if child.text != None:
                    issue_name_formatted += f' {child.text}.'

            issue_name_formatted = issue_name_formatted.rstrip('.')

    if 'issue' in append_VL_ID:
        issue_name_formatted += f'.{get_id(issue_url)}'

    return issue_name_formatted

get_id = lambda string: string.split('=')[-1]

def get_formatted_page_name(page_id, page_name_format, newspaper_series_url, newspaper_series_name, newspaper_url,
                            newspaper_names, year_url, year_date_format, issue_url, issue_date_format, issue_name_granularity, fmt, append_VL_ID):
    """Get file name for a page according to specified format. Use issue to fetch necessary metadata."""
    issue_root = get_root(issue_url)

    if fmt.lower() == 'download':
        pages = issue_root.xpath(f'//mets:structMap[@TYPE="PHYSICAL"]/mets:div', namespaces=ns_map)[0]
        first_page = pages[0].attrib['ORDER']
        last_page = pages[-1].attrib['ORDER']
        page_no = f'{first_page}-{last_page}'
    else:
        page_no = issue_root.xpath(f'//mets:div[@ID="phys{page_id}"]/@ORDER', namespaces=ns_map)[0]

    formatted_page_name = page_name_format.replace('PAGE_NO', page_no)

    if 'PAGE_ID' in page_name_format:
        formatted_page_name = formatted_page_name.replace('PAGE_ID', page_id)
    if 'ISSUE_NAME' in page_name_format:
        issue_name = get_issue_name(issue_url, issue_date_format, issue_name_granularity, append_VL_ID)
        formatted_page_name = formatted_page_name.replace('ISSUE_NAME', issue_name)
    if 'YEAR_NAME' in page_name_format:
        year_name = get_year_name(year_url, year_date_format, append_VL_ID)
        formatted_page_name = formatted_page_name.replace('YEAR_NAME', year_name)
    if 'NEWSPAPER_NAME' in page_name_format:
        newspaper_name = get_newspaper_name(newspaper_url, newspaper_names, append_VL_ID)
        formatted_page_name = formatted_page_name.replace('NEWSPAPER_NAME', newspaper_name)
    if 'NEWSPAPER_SERIES_NAME' in page_name_format:
        newspaper_series_name = get_newspaper_series_name(newspaper_series_url, newspaper_series_name, append_VL_ID)
        formatted_page_name = formatted_page_name.replace('NEWSPAPER_SERIES_NAME', newspaper_series_name)

    if 'page' in append_VL_ID:
        formatted_page_name += f'.{page_id}'

    return formatted_page_name


def create_download_path(newspaper_series_url, newspaper_series_name, newspaper_url, newspaper_names,
                         year_url, year_date_format, issue_url, issue_date_format, issue_name_granularity, fmt, folder_structure,
                         base, append_VL_ID):
    newspaper_series = get_newspaper_series_name(newspaper_series_url, newspaper_series_name, append_VL_ID)
    newspaper = get_newspaper_name(newspaper_url, newspaper_names, append_VL_ID)
    year = get_year_name(year_url, year_date_format, append_VL_ID)
    issue = get_issue_name(issue_url, issue_date_format, issue_name_granularity, append_VL_ID)

    download_path = ""
    for element in folder_structure:
        download_path += f'{eval(element)}/'
    download_path = download_path.rstrip('/')

    try:
        makedirs(f'{download_path}')
    except FileExistsError:
        pass

    return download_path


def download_page(page_link, download_path, page_name_format, newspaper_series_url, newspaper_series_name,
                                            newspaper_url, newspaper_names, year_url, year_date_format, issue_url, issue_date_format,
                                            issue_name_granularity, fmt, append_VL_ID):
    """Extract name and filetype from link to newspaper page. Then save at specified folder."""

    r = s.get(page_link)
    page_id = page_link.split('/')[-1]

    # get page name according to desired format
    if page_name_format == 'PAGE_ID':
        page_name = page_id
    else:
        page_name = get_formatted_page_name(page_id, page_name_format, newspaper_series_url, newspaper_series_name,
                                            newspaper_url, newspaper_names, year_url, year_date_format,
                                            issue_url, issue_date_format, issue_name_granularity, fmt, append_VL_ID)

    # get rid of encoding if provided in r.headers; if not provided: split still works, then guess extension
    filetype = guess_extension(r.headers['Content-Type'].split(';')[0])
    filename = page_name + filetype

    # throw warning if file will be overwritten
    if path.isfile(f'{download_path}/{filename}'):
        print(f'Warning: {filename} already existed and was overwritten')

    # save file
    with open(f'{download_path}/{filename}', 'wb') as file:
        file.write(r.content)


for newspaper_url in get_newspapers_from_newspaper_series(newspaper_series_url):
    for year_url in get_years_from_newspaper(newspaper_url)[:2]:
        for issue_url in get_issues_from_year(year_url):
            for fmt in formats:
                download_path = create_download_path(newspaper_series_url, newspaper_series_name, newspaper_url,
                                                     newspaper_names, year_url, year_name_format, issue_url, issue_name_format,
                                                     issue_name_granularity, fmt, folder_structure, base, append_VL_ID)
                for page_link in get_pages_from_issue(issue_url, fmt=fmt):
                    download_page(page_link, download_path, page_name_format, newspaper_series_url, newspaper_series_name,
                                  newspaper_url, newspaper_names, year_url, year_name_format, issue_url, issue_name_format, issue_name_granularity,
                                  fmt, append_VL_ID)

                # Remove empty directories. This can happen if one format (e.g. Fulltext) is not present for an issue.
                try:
                    rmdir(download_path)
                except Exception:
                    pass

Vorwort¶

Einleitung¶

Idee¶

Code¶

Namenskonfiguration¶

Funktionsdefinitionen¶

Download¶