Skip to content

Commit

Permalink
Merge branch 'OSeMOSYS:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@HauHe
HauHe authored Jun 24, 2021
2 parents fdee300 + 6d86dd0 commit 029d19b
Show file tree
Hide file tree
Showing 5 changed files with 432 additions and 140 deletions.
84 changes: 84 additions & 0 deletions .zenodo.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
{
"license": "MIT",
"upload_type": "software",
"creators": [
{
"name": "Mark Howells",
"orcid": "0000-0001-6419-4957",
"affiliation": "Imperial College London; Loughborough University Department of Social Sciences"
},
{
"name": "Manuel Welsch",
"affiliation": "International Atomic Energy Agency"
},
{
"name": "Constantinos Taliotis",
"orcid": "0000-0003-4022-5506",
"affiliation": "The Cyprus Institute"
},
{
"name": "Adrian Lefvert",
"orcid": "0000-0001-8587-4054",
"affiliation": "KTH Royal Institute of Technology"
},
{
"name": "Igor Tatarewicz",
"affiliation": "Institute of Environmental Protection - National Research Institute / National Centre for Emissions Management (KOBiZE)"
},
{
"name": "Tom Alfstad",
"affiliation": "United Nations Department of Economic and Social Affairs"
},
{
"name": "Nawfal Saadi",
"orcid": "0000-0001-8923-7431",
"affiliation": "Enel Green Power"
},
{
"name": "Francesco Gardumi",
"affiliation": "KTH Royal Institute of Technology",
"orcid": "0000-0001-8371-9325"
},
{
"name": "Vignesh Sridharan",
"affiliation": "KTH Royal Institute of Technology",
"orcid": "0000-0003-0764-2615"
},
{
"name": "Agnese Beltramo",
"affiliation": "KTH Royal Institute of Technology",
"orcid": "0000-0001-6591-3028"
},
{
"name": "Nandi Moksnes",
"affiliation": "KTH Royal Institute of Technology",
"orcid": "0000-0002-8641-564X"
},
{
"name": "Taco Niet",
"affiliation": "Simon Fraser University",
"orcid": "0000-0003-0266-2705"
},
{
"name": "Abhishek Shivakumar",
"affiliation": "United Nations Department of Economic and Social Affairs",
"orcid": "0000-0002-2535-4134"
},
{
"name": "Roberto Heredia Fonseca",
"affiliation": "KTH Royal Institute of Technology",
"orcid": "0000-0003-3947-8725"
},
{
"name": "Will Usher",
"affiliation": "KTH Royal Institute of Technology",
"orcid": "0000-0001-9367-1791"
},
{
"name": "Christoph Muschner",
"affiliation": "KTH Royal Institute of Technology",
"orcid": "0000-0001-8144-5260"
}
],
"access_right": "open"
}
131 changes: 72 additions & 59 deletions scripts/preprocess_data.py
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,25 @@
"""Pre-process OSeMOSYS data file to reduce matrix generation time
This script pre-processes an OSeMOSYS input data file by adding lines that list
commodity-technology-mode combinations that data is provided for. Pre-processing
a data file before starting a model run significantly reduces the time taken
for matrix generation.
This script pre-processes an OSeMOSYS input data file by adding lines that list
commodity-technology-mode combinations that data is provided for. Pre-processing
a data file before starting a model run significantly reduces the time taken
for matrix generation.
Pre-processing consists of the following steps:
1. Reading the ``InputActivityRatio`` and ``OutputActivityRatio`` sections of the
data file to identify commodity-technology-mode combinations that data has been
1. Reading the ``InputActivityRatio`` and ``OutputActivityRatio`` sections of the
data file to identify commodity-technology-mode combinations that data has been
explicitly provided for.
2. Adding a set entry for each commodity that lists all technology-mode combinations
that are associated with it.
3. Values from the ``InputActivityRatios`` and ``OutputActivityRatios`` sections are
2. Adding a set entry for each commodity that lists all technology-mode combinations
that are associated with it.
3. Values from the ``InputActivityRatios`` and ``OutputActivityRatios`` sections are
added to the sets ``MODExTECHNOLOGYperFUELin`` and ``MODExTECHNOLOGYperFUELout`` respectively.
4. Values from the ``TechnologyToStorage`` and ``TechnologyFromStorage`` sections
4. Values from the ``TechnologyToStorage`` and ``TechnologyFromStorage`` sections
are added to the sets ``MODExTECHNOLOGYperSTORAGEto`` and ``MODExTECHNOLOGYperSTORAGEfrom`` respectively.
5. All values for technology-mode combinations are added to the sets
5. All values for technology-mode combinations are added to the sets
``MODEperTECHNOLOGY``.
In order to start a model run with a pre-processed data file, the following sets
In order to start a model run with a pre-processed data file, the following sets
need to be introduced to its associated OSeMOSYS model file::
set MODEperTECHNOLOGY{TECHNOLOGY} within MODE_OF_OPERATION;
Expand Down Expand Up @@ -70,20 +70,20 @@ def main(data_format, data_infile, data_outfile):
storage_from = []
emission_table = []

param_check = ['OutputActivityRatio', 'InputActivityRatio', 'TechnologyToStorage', 'TechnologyFromStorage', 'EmissionActivityRatio']
params_to_check = ['OutputActivityRatio', 'InputActivityRatio', 'TechnologyToStorage', 'TechnologyFromStorage', 'EmissionActivityRatio']

with open(data_infile, 'r') as f:
for line in f:
if parsing_year:
year_list += [line.strip()] if line.strip() not in ['', ';'] else []
if parsing_fuel:
fuel_list += [line.strip()] if line.strip() not in ['', ';'] else []
fuel_list += [line.strip()] if line.strip() not in ['', ';'] else []
if parsing_tech:
tech_list += [line.strip()] if line.strip() not in ['', ';'] else []
tech_list += [line.strip()] if line.strip() not in ['', ';'] else []
if parsing_storage:
storage_list += [line.strip()] if line.strip() not in ['', ';'] else []
storage_list += [line.strip()] if line.strip() not in ['', ';'] else []
if parsing_mode:
mode_list += [line.strip()] if line.strip() not in ['', ';'] else []
mode_list += [line.strip()] if line.strip() not in ['', ';'] else []
if parsing_emission:
emission_list += [line.strip()] if line.strip() not in ['', ';'] else []

Expand All @@ -92,12 +92,12 @@ def main(data_format, data_infile, data_outfile):
year_list = line.split(' ')[3:-1]
else:
parsing_year = True
if line.startswith('set COMMODITY'): # Extracts list of COMMODITIES from data file. Some models use FUEL instead.
if line.startswith('set COMMODITY'): # Extracts list of COMMODITIES from data file. Some models use FUEL instead.
if len(line.split('=')[1]) > 1:
fuel_list = line.split(' ')[3:-1]
else:
parsing_fuel = True
if line.startswith('set FUEL'): # Extracts list of FUELS from data file. Some models use COMMODITIES instead.
if line.startswith('set FUEL'): # Extracts list of FUELS from data file. Some models use COMMODITIES instead.
if len(line.split('=')[1]) > 1:
fuel_list = line.split(' ')[3:-1]
else:
Expand Down Expand Up @@ -150,13 +150,13 @@ def main(data_format, data_infile, data_outfile):
values = line.rstrip().split(' ')[1:]
mode = line.split(' ')[0]

if param_current == 'OutputActivityRatio':
if param_current == 'OutputActivityRatio':
data_out.append(tuple([fuel, tech, mode]))
for i in range(0, len(years)):
output_table.append(tuple([tech, fuel, mode, years[i], values[i]]))

if param_current == 'InputActivityRatio':
data_inp.append(tuple([fuel, tech, mode]))
data_inp.append(tuple([fuel, tech, mode]))

data_all.append(tuple([tech, mode]))

Expand Down Expand Up @@ -186,52 +186,61 @@ def main(data_format, data_infile, data_outfile):
if data_format == 'otoole':
with open(data_infile, 'r') as f:
for line in f:
details = line.split(' ')
if line.startswith(";"):
parsing = False
if parsing:
if len(line.split(' ')) > 1:
if len(details) > 1:
if param_current == 'OutputActivityRatio':
tech = line.split(' ')[1].strip()
fuel = line.split(' ')[2].strip()
mode = line.split(' ')[3].strip()
year = line.split(' ')[4].strip()
value = line.split(' ')[5].strip()
tech = details[1].strip()
fuel = details[2].strip()
mode = details[3].strip()
year = details[4].strip()
value = details[5].strip()

data_out.append(tuple([fuel, tech, mode]))
output_table.append(tuple([tech, fuel, mode, year, value]))
if float(value) != 0.0:
data_out.append(tuple([fuel, tech, mode]))
output_table.append(tuple([tech, fuel, mode, year, value]))
data_all.append(tuple([tech, mode]))

if param_current == 'InputActivityRatio':
tech = line.split(' ')[1].strip()
fuel = line.split(' ')[2].strip()
mode = line.split(' ')[3].strip()

data_inp.append(tuple([fuel, tech, mode]))

data_all.append(tuple([tech, mode]))
tech = details[1].strip()
fuel = details[2].strip()
mode = details[3].strip()
value = details[5].strip()
if float(value) != 0.0:
data_inp.append(tuple([fuel, tech, mode]))
data_all.append(tuple([tech, mode]))

if param_current == 'TechnologyToStorage':
tech = line.split(' ')[1].strip()
storage = line.split(' ')[2].strip()
mode = line.split(' ')[3].strip()

storage_to.append(tuple([storage, tech, mode]))
tech = details[1].strip()
storage = details[2].strip()
mode = details[3].strip()
value = details[4].strip()
if value > 0.0:
storage_to.append(tuple([storage, tech, mode]))
data_all.append(tuple([storage, mode]))

if param_current == 'TechnologyFromStorage':
tech = line.split(' ')[1].strip()
storage = line.split(' ')[2].strip()
mode = line.split(' ')[3].strip()

storage_from.append(tuple([storage, tech, mode]))
tech = details[1].strip()
storage = details[2].strip()
mode = details[3].strip()
value = details[4].strip()
if value > 0.0:
storage_from.append(tuple([storage, tech, mode]))
data_all.append(tuple([storage, mode]))

if param_current == 'EmissionActivityRatio':
tech = line.split(' ')[1].strip()
emission = line.split(' ')[2].strip()
mode = line.split(' ')[3].strip()

emission_table.append(tuple([emission, tech, mode]))

if any(param in line for param in param_check):
param_current = line.split(' ')[-2]
tech = details[1].strip()
emission = details[2].strip()
mode = details[3].strip()
value = details[5].strip()
if float(value) != 0.0:
emission_table.append(tuple([emission, tech, mode]))
data_all.append(tuple([tech, mode]))

if any(param in line for param in params_to_check):
param_current = details[-2]
parsing = True

data_out = list(set(data_out))
Expand Down Expand Up @@ -277,8 +286,12 @@ def file_output_function(if_dict, str_dict, set_list, set_name, extra_char):
line = line.replace('),',')').replace('[(',' (').replace(')]',')').replace("'","")
else:
line = set_name + str(each) + ']:='

file_out.write(line + ';' + '\n')

storage_list_len = {'otoole': 0,
'momani': 1}

# Append lines at the end of the data file
with open(data_outfile, 'w') as file_out: # 'a' to open in 'append' mode

Expand All @@ -288,11 +301,11 @@ def file_output_function(if_dict, str_dict, set_list, set_name, extra_char):
file_output_function(dict_inp, dict_inp, fuel_list, 'set MODExTECHNOLOGYperFUELin[', '')
file_output_function(dict_all, dict_all, tech_list, 'set MODEperTECHNOLOGY[', '*')

if len(storage_list) > 1:
file_output_function(dict_stt, dict_out, storage_list, 'set MODExTECHNOLOGYperSTORAGEto[', '')
file_output_function(dict_stf, dict_out, storage_list, 'set MODExTECHNOLOGYperSTORAGEfrom[', '*')
if len(storage_list) > storage_list_len[data_format]:
file_output_function(dict_stt, dict_stt, storage_list, 'set MODExTECHNOLOGYperSTORAGEto[', '')
file_output_function(dict_stf, dict_stf, storage_list, 'set MODExTECHNOLOGYperSTORAGEfrom[', '')

if len(emission_list) > 1:
if len(emission_list) > 0:
file_output_function(dict_emi, dict_emi, emission_list, 'set MODExTECHNOLOGYperEMISSION[', '')

file_out.write('end;')
Expand All @@ -308,4 +321,4 @@ def file_output_function(if_dict, str_dict, set_list, set_name, extra_char):
data_format = sys.argv[1]
data_infile = sys.argv[2]
data_outfile = sys.argv[3]
main(data_format, data_infile, data_outfile)
main(data_format, data_infile, data_outfile)
Loading

0 comments on commit 029d19b

Please sign in to comment.