amiberry_xml_refresher.py

import datetime
import math
import os
from pathlib import Path
import shutil
import sys
from utils import text_utils
import xml.etree.ElementTree as etree

# =======================================
# Functions
# =======================================
# Get a value from a file
def value_list(in_file, game_name):
    file_name = 'settings/' + in_file

    if os.path.isfile(file_name) is False:
        return ''

    with open(file_name) as f:
        content = f.readlines()
        content = [x.strip() for x in content]
    f.close()

    answer = ''

    for this_line in content:
        if not this_line == '':
            this_word = this_line.split()
            if this_word[0] == game_name:
                answer = this_word[1]
                break

    return answer

# Ensure a game package is set within a file
def check_list(in_file, game_name):
    temp_game = game_name
    file_name = 'settings/' + in_file

    if os.path.isfile(file_name) is False:
        return False

    with open(file_name) as f:
        content = f.readlines()
        content = [x.strip() for x in content]
    f.close()

    answer = False

    for this_line in content:
        if this_line == temp_game:
            answer = True
            break

    return answer

# XML sorting
def sortchildrenby(parent, attr):
    parent[:] = sorted(parent, key=lambda child: child.get(attr).lower())

# =======================================
# main section starts here...
# =======================================
script_version = '0.9'
print()
print(
    text_utils.FontColours.BOLD + text_utils.FontColours.OKBLUE + "HoraceAndTheSpider and osvaldolove's" + text_utils.FontColours.ENDC + text_utils.FontColours.BOLD +
    ' Amiberry XML Refresher' + text_utils.FontColours.ENDC + text_utils.FontColours.OKGREEN + ' ('+script_version+')' + text_utils.FontColours.ENDC)
print()

whdbfile = 'whdload_db.xml'
whdbtmp = 'whdload_db.xml.tmp'
whdbbak = 'whdload_db.xml.bak'
xmlerrorlog = 'xml_error_syntax.log'

# get whdbfile size before modification
whdsize = Path(whdbfile).stat().st_size

# get whdbfile atime and mtime before modification
# script updates packages newer than whdload_db.xml mtime
whdtime = os.stat(whdbfile)

# Backup
if not os.path.isfile(whdbbak) or whdsize >= os.stat(whdbbak).st_size:
    shutil.copy2(whdbfile, whdbbak)
    whdbaksize = Path(whdbbak).stat().st_size

# =======================================
# Start XML generation
# =======================================
XML_HEADER = '<whdbooter timestamp="' + datetime.datetime.now().strftime("%Y-%m-%d at %H:%M:%S") + '">'
XML = ''
XML_FOOTER = chr(10) + '</whdbooter>'

# parse XML and validation
try:
    root = etree.parse(whdbfile)
    print('XML format: OK')

# check for XML syntax errors
except etree.XMLSyntaxError as err:
    print('XML Syntax Error, check', xmlerrorlog)
    with open(xmlerrorlog, 'w') as error_log_file:
        error_log_file.write(str(err.error_log))

except:
    print('Unknown error with XML file.')

# get XML root
root = etree.parse(whdbfile).getroot()
total_item = len(root) 
count = 0

# brutal way: extract then reinject data into the XML.
# need to think of a smarter to do that // perf.
for item in root.findall('game'):
    count += 1
    file_name = item.get('filename')
    ArchiveSHA = item.get('sha1')

    full_game_name = item.find('name').text
    sub_path = item.find('subpath').text
    variant_uuid = item.find('variant_uuid').text
    slave_count = item.find('slave_count').text
    slave_default = item.find('slave_default').text
    slave_libraries = item.find('slave_libraries').text
    # Extract the 'slave' block
    SLAVE_XML = ''
    SXML = []
    for slave in item.findall('.slave'):
        SXML.append(etree.tostring(slave).decode()) # convert to str
        SLAVE_XML = ''.join(SXML)

    # Attempt to fix empty name or subpath
    if not full_game_name or not sub_path:
        full_game_name = file_name.replace('_',' ')
        get_sub_path = text_utils.left(slave_default,len(slave_default) - len('.slave'))
        if get_sub_path.find('\\') > -1:
            sub_path = get_sub_path.split('\\', 1)[0]
        else:
            sub_path = get_sub_path
                
    # =======================================
    # DISPLAY SETTINGS

    # Amiberry 3.2+: HEIGHT can be any value yet let's stick to the following
    # values to keep things tidy and easy to maintain.
    # Possible values: 400, 432, 480, 512, 524, 540, 568
    # default: AUTOHEIGHT
    listheights = ['400', '432', '480', '512', '524', '540', '568']
    HW_HEIGHT = ''

    for possibleheight in listheights:
        if check_list('Screen_Height_'+possibleheight+'.txt', sub_path) is True:
            HW_HEIGHT = possibleheight
            break

    # Amiberry 3.2+: WIDTH can be any value yet let's stick to the following
    # values to keep things tidy and easy to maintain.
    # Possible values: 640, 704, 720
    # default: 720
    listwidths = ['640', '704']
    HW_WIDTH = '720'

    for possiblewidth in listwidths:
        if check_list('Screen_Width_'+possiblewidth+'.txt', sub_path) is True:
            HW_WIDTH = possiblewidth
            break

    # centering 
    # default: enabled
    HW_H_CENTER = 'SMART'
    if check_list('Screen_NoCenter_H.txt', sub_path) is True:
        HW_H_CENTER = 'NONE'

    HW_V_CENTER = 'SMART'
    if check_list('Screen_NoCenter_V.txt', sub_path) is True:
        HW_V_CENTER = 'NONE'

    # offset
    # default: 0 (for both horizontal and vertical)
    offset_h = value_list('Screen_Offset_H.txt', sub_path)
    offset_v = value_list('Screen_Offset_V.txt', sub_path)

    min_offset_h = -60
    max_offset_h = 60

    if offset_h.lstrip('-').isnumeric():
        HW_H_OFFSET = int(offset_h)
        if min_offset_h <= HW_H_OFFSET <= max_offset_h:
            pass
        elif HW_H_OFFSET < min_offset_h:
            HW_H_OFFSET = min_offset_h
        elif HW_H_OFFSET > max_offset_h:
            HW_H_OFFSET = max_offset_h
    else:
        HW_H_OFFSET = ''

    min_offset_v = -20
    max_offset_v = 20

    if offset_v.lstrip('-').isnumeric():
        HW_V_OFFSET = int(offset_v)
        if min_offset_v <= HW_V_OFFSET <= max_offset_v:
            pass
        elif HW_V_OFFSET < min_offset_v:
            HW_V_OFFSET = min_offset_v
        elif HW_V_OFFSET > max_offset_v:
            HW_V_OFFSET = max_offset_v
    else:
        HW_V_OFFSET = ''

    # NTSC
    HW_NTSC = ''
    if check_list('Chipset_ForceNTSC.txt', sub_path) is True:
        HW_NTSC = 'TRUE'       
    elif full_game_name.find('NTSC') > -1:
        HW_NTSC = 'TRUE' 
                            
    # =======================================
    # CONTROL SETTINGS
    # mouse / mouse 2 / CD32

    use_mouse1 = check_list('Control_Port0_Mouse.txt', sub_path)
    use_mouse2 = check_list('Control_Port1_Mouse.txt', sub_path)
    use_cd32_pad = check_list('Control_CD32.txt', sub_path)

    # =======================================
    # MEMORY SETTINGS
    # Let's limit possible Z3 values to 128Mb.
    # Amiberry 5+: 8Mb of fast RAM/Z2 set as default
    # Default: 2Mb Chip / 8Mb Z2 / 0Mb Z3

    for i in range(0, 8): # No more than 128MB
        z3_ram = int(math.pow(2, i))
        if check_list('Memory_Z3Ram_' + str(z3_ram) + '.txt', sub_path) is True:
            HW_24BIT = 'FALSE'
            break
        else:
            z3_ram = 0
            HW_24BIT = ''

    # =======================================
    # CHIPSET SETTINGS

    # sprite collisions
    # Default: Playfield
    # can't find a single case requiring value different than default.

    # blitter    
    # Default: Wait for Blitter
    HW_BLITS = ''        
    if check_list('Chipset_ImmediateBlitter.txt', sub_path) is True:
        HW_BLITS = 'IMMEDIATE'

    # fast copper
    # Default: False
    HW_FASTCOPPER = ''
    if check_list('Chipset_FastCopper.txt', sub_path) is True:
        HW_FASTCOPPER = 'TRUE'

    # =======================================
    # CPU SETTINGS

    # clock speed (MHz)
    # Default: 7 for non-AGA / 14 for AGA
    HW_SPEED = ''
    if check_list('CPU_ClockSpeed_25.txt', sub_path) is True:
        HW_SPEED = '25'
    if check_list('CPU_ClockSpeed_Max.txt', sub_path) is True:
        HW_SPEED = 'MAX'

    # cpu model
    # Default: 68000 for non-AGA / 68020 for AGA

    # 24 bit addressing 
    # Default: True / you can set Z3 separately
             
    # compatible CPU 
    # Default: True
    HW_CPUCOMP = ''
    if check_list('CPU_NoCompatible.txt', sub_path) is True:
        HW_CPUCOMP = 'FALSE'
                    
    # CPU cycle exact
    # Available only with 68000 CPU
    # Default: False 
    HW_CPUEXACT = ''
    if check_list('CPU_CycleExact.txt', sub_path) is True:
        HW_CPUEXACT = 'TRUE'

    # JIT Cache
    # Default: False
    HW_JIT = ''
    if check_list('CPU_ForceJIT.txt',sub_path) is True:
        HW_JIT = 'TRUE'

    # CHIPSET
    # Default: Full ECS
    HW_CHIPSET = ''
    if file_name.find('_AGA') > -1:
        HW_CHIPSET = 'AGA'                                       
    if file_name.find('_CD32') > -1:
        HW_CHIPSET = 'AGA'
        use_cd32_pad = True
    if check_list('Chipset_AGA.txt',sub_path) is True:
        HW_CHIPSET = 'AGA'

    # ================================
    # building hardware section

    hardware = ''
    if HW_BLITS != '':
        hardware += chr(10) + ('BLITTER=') + HW_BLITS

    if HW_CHIPSET != '':
        hardware += chr(10) + ('CHIPSET=') + HW_CHIPSET
                    
    if HW_SPEED != '':
        hardware += chr(10) + ('CLOCK=') + HW_SPEED

    if HW_24BIT != '':
        hardware += chr(10) + ('CPU_24BITADDRESSING=') + HW_24BIT

    if HW_CPUCOMP != '':
        hardware += chr(10) + ('CPU_COMPATIBLE=') + HW_CPUCOMP

    if HW_CPUEXACT != '':
        hardware += chr(10) + ('CPU_EXACT=') + HW_CPUEXACT

    if HW_FASTCOPPER != '':
        hardware += chr(10) + ('FAST_COPPER=') + HW_FASTCOPPER

    if HW_JIT != '':
        hardware += chr(10) + ('JIT=') + HW_JIT

    if HW_NTSC != '':
        hardware += chr(10) + ('NTSC=') + HW_NTSC

    if use_mouse1 == True:
        hardware += chr(10) + ('PRIMARY_CONTROL=MOUSE')
    else:
        hardware += chr(10) + ('PRIMARY_CONTROL=JOYSTICK')

    if use_mouse1 == True:
        hardware += chr(10) + ('PORT0=MOUSE')
    elif use_cd32_pad == True:
        hardware += chr(10) + ('PORT0=CD32')
    else:
        hardware += chr(10) + ('PORT0=JOY')

    if use_mouse2 == True:
        hardware += chr(10) + ('PORT1=MOUSE')
    elif use_cd32_pad == True:
        hardware += chr(10) + ('PORT1=CD32')
    else:
        hardware += chr(10) + ('PORT1=JOY')

    # Screen: size, auto-height/crop
    # Disable AUTOHEIGHT and set HEIGHT only when there's HEIGHT
    if HW_HEIGHT != '':
        HW_AUTO_HEIGHT = 'FALSE'
        hardware += chr(10) + ('SCREEN_AUTOHEIGHT=') + HW_AUTO_HEIGHT
        hardware += chr(10) + ('SCREEN_HEIGHT=') + HW_HEIGHT
    else:
        HW_AUTO_HEIGHT = 'TRUE'
        hardware += chr(10) + ('SCREEN_AUTOHEIGHT=') + HW_AUTO_HEIGHT

    if HW_WIDTH != '720' or HW_AUTO_HEIGHT == 'FALSE':
        hardware += chr(10) + ('SCREEN_WIDTH=') + HW_WIDTH

    # H_CENTER only if there's no H_OFFSET
    if HW_H_CENTER == 'SMART' and HW_H_OFFSET != '':
        HW_H_CENTER = 'NONE'

    hardware += chr(10) + ('SCREEN_CENTERH=') + HW_H_CENTER

    # V_CENTER only if there's no V_OFFSET
    if HW_V_CENTER == 'SMART' and HW_V_OFFSET != '':
        HW_V_CENTER = 'NONE'

    hardware += chr(10) + ('SCREEN_CENTERV=') + HW_V_CENTER

    if HW_H_OFFSET != '':
        hardware += chr(10) + ('SCREEN_OFFSETH=') + str(HW_H_OFFSET)

    if HW_V_OFFSET != '':
        hardware += chr(10) + ('SCREEN_OFFSETV=') + str(HW_V_OFFSET)

    if z3_ram != 0:
        hardware += chr(10) + ('Z3_RAM=') + str(z3_ram)

    # custom controls
    custom_file = 'customcontrols/' + sub_path
    custom_text = ''
                                
    # remove any items which are not amiberry custom settings
    if os.path.isfile(custom_file) == True:
        with open(custom_file, 'r') as f:
            customsettings_content = f.readlines()
        f.close()

        for this_line in customsettings_content:
            if this_line.find('amiberry_custom') > -1 and '\n' in this_line:
                custom_text += chr(9) + chr(9) + chr(9) + this_line
            elif this_line.find('amiberry_custom') > -1 and not '\n' in this_line:
                custom_text += chr(9) + chr(9) + chr(9) + this_line + chr(10)

    # external libraries (eg. xpk, required for Dungeon Master)
    extra_libs = 'False'
    if check_list('WHD_Libraries.txt', sub_path) is True:
        extra_libs = 'True'

    # select default slave if defined
    check_slave = value_list('WHD_DefaultSlave.txt', file_name)
    if check_slave != '':
        slave_default = check_slave

    # generate XML
    XML += chr(10) + chr(9) + '<game filename="' + file_name.replace('&', '&amp;') + '" sha1="' + ArchiveSHA + '">' + chr(10)
    XML += chr(9) + chr(9) + '<name>' + full_game_name.replace('&', '&amp;') + '</name>' + chr(10)
    XML += chr(9) + chr(9) + '<subpath>' + sub_path.replace('&', '&amp;') + '</subpath>' + chr(10)
    XML += chr(9) + chr(9) + '<variant_uuid>' + variant_uuid + '</variant_uuid>' + chr(10)
    XML += chr(9) + chr(9) + '<slave_count>' + slave_count + '</slave_count>' + chr(10)
    XML += chr(9) + chr(9) + '<slave_default>' + slave_default.replace('&', '&amp;') + '</slave_default>' + chr(10)
    XML += chr(9) + chr(9) + '<slave_libraries>' + extra_libs  + '</slave_libraries>' + chr(10)
    XML += chr(9) + chr(9) + SLAVE_XML
    XML += '<hardware>'
    XML += hardware.replace(chr(10), chr(10) + chr(9) + chr(9) + chr(9))
    XML += chr(10) + chr(9) + chr(9) + '</hardware>' + chr(10)

    if len(custom_text) > 0:
        XML += chr(9) + chr(9) + '<custom_controls>' + chr(10) + custom_text + chr(9) + chr(9) + '</custom_controls>' + chr(10)
                
    XML += chr(9) + '</game>'

    # Show progress on command-line
    cur_pct = (count / total_item)
    pkg_percent = '{:.0%}'.format(cur_pct)
    print('Refreshing XML.',total_item,'packages found:',pkg_percent,'done', end='\r', flush=True)

# =======================================
# XML snippets
# for recent games
# =======================================
print()
print('Adding external XML snippets')

snippet_dir = 'snippets/'
snippet_text =  ''

# add fake root to 'old' XML to get valid xml
xml_old_snippet = '<root>' + XML + '</root>'
snipoldroot = etree.fromstring(xml_old_snippet)

for snippet_file in os.listdir(snippet_dir):
    xmlsnip = os.path.join(snippet_dir,snippet_file)
    with open(xmlsnip, 'r') as snippets_content:

        # add fake root to 'snippets' to get valid xml
        xml_snippet = '<root>' + snippets_content.read() + '</root>'
        sniproot = etree.fromstring(xml_snippet)

        # check if snippet's element has no duplicate then update it
        for snipgame in sniproot.findall('game'):
            snippet_filename = snipgame.get('filename')
            snippet_sha1 = snipgame.get('sha1')

            # check for sha1 in snippets
            if snippet_sha1 in snippet_text:
                continue

            # check for sha1 in 'old' XML
            # if found delete it - it will be re-added below
            # this way we would always get the latest version
            if snippet_sha1 in XML:
                for old_elem in snipoldroot.findall('.//game[@sha1="{value}"]'.format(value=snippet_sha1)):
                    snipoldroot.remove(old_elem)

            # add element from snippet
            print('adding ' + snippet_filename)
            snipstr = etree.tostring(snipgame).decode().strip()
            if snipstr.startswith('\t'):
                snippet_text += snipstr + chr(10)
            else:
                snippet_text += chr(9) + snipstr + chr(10)

# return a string and delete the fake root
XML = etree.tostring(snipoldroot).decode()
XML = XML.replace('<root>', '').replace('</root>', '')

if len(snippet_text) > 0:
   XML += chr(10) + snippet_text + chr(9)
                
# =======================================
# XML is complete, let's put it all together
# =======================================
XML = XML_HEADER + XML + XML_FOOTER

# =======================================
# write down XML file
# =======================================
print('Generating XML File')
text_file = open(whdbtmp, 'w+')
text_file.write(XML)
text_file.close()

# =======================================
# Line Squasher
# * line(s) matching specified characters will be deleted 
# * ensure only ASCII characters are written
#
# eg.:
# offtext = ['FAST_RAM=8', '\t\t\n']
# =======================================
offtext = []

with open(whdbtmp, 'r') as deloffset:
    olines = deloffset.readlines()

with open(whdbtmp, 'w') as nomoreoffset:
    for line in olines:
        # remove blank lines
        if line.rstrip():
            # ensure only ASCII characters
            if not any(offset in line for offset in offtext) and all(ord(ch) < 128 for ch in line):
               nomoreoffset.write(line)

# =======================================
# Sorting elements 
# Not required yet easier to debug
# =======================================
print('Sorting XML File')
tree = etree.parse(whdbtmp)
parent = tree.getroot()
sortchildrenby(parent, 'filename')
tree.write(whdbfile, encoding='utf-8', xml_declaration=True)

# =======================================
# Restore whdbfile mtime
# Ensure new updated packages won't be missed
# =======================================
os.utime(whdbfile, (whdtime.st_atime, whdtime.st_mtime))

# =======================================
# Cleaning
# =======================================
# Remove the tmp whdload_db.xml
os.remove(whdbtmp)

# get whdbfile size after modification
whdsize_after = Path(whdbfile).stat().st_size

if whdsize_after > 0 and whdsize_after >= whdsize:
    os.remove(whdbbak)

# =======================================
print('Bye!')