From 5813d12237a747213f7a012605d17bcc59010279 Mon Sep 17 00:00:00 2001 From: Kai Nagel Date: Fri, 24 May 2024 14:10:27 +0200 Subject: [PATCH 1/5] towards rail scraping --- .../bvwp/users/kn/RunLocalRailScrapingKN.java | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java diff --git a/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java b/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java new file mode 100644 index 0000000..e05973d --- /dev/null +++ b/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java @@ -0,0 +1,155 @@ +package org.tub.vsp.bvwp.users.kn; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.tub.vsp.bvwp.BvwpUtils; +import org.tub.vsp.bvwp.data.Headers; +import org.tub.vsp.bvwp.data.container.analysis.StreetAnalysisDataContainer; +import org.tub.vsp.bvwp.data.type.Einstufung; +import org.tub.vsp.bvwp.io.StreetCsvWriter; +import org.tub.vsp.bvwp.plot.MultiPlotUtils; +import org.tub.vsp.bvwp.scraping.RailScraper; +import org.tub.vsp.bvwp.scraping.Scraper; +import org.tub.vsp.bvwp.scraping.StreetScraper; +import tech.tablesaw.api.Row; +import tech.tablesaw.api.Table; +import tech.tablesaw.plotly.components.Figure; +import tech.tablesaw.plotly.display.Browser; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Paths; +import java.text.NumberFormat; +import java.util.*; + +public class RunLocalRailScrapingKN{ + private static final Logger logger = LogManager.getLogger( RunLocalRailScrapingKN.class ); + + public static void main(String[] args) throws IOException{ + Locale.setDefault( Locale.US ); + + RailScraper scraper = new RailScraper(); + + logger.info( "Starting scraping" ); + + String filePath = "../../shared-svn/"; + Map constructionCostsByProject = BvwpUtils.getConstructionCostsFromTudFile(filePath ); + + + // yyyy man könnte (sollte?) den table in den StreetAnalysisDataContainer mit hinein geben, und die Werte gleich dort eintragen. kai, feb'24 + List allStreetBaseData = scraper + .stream() + .map(streetBaseDataContainer -> new StreetAnalysisDataContainer( + streetBaseDataContainer, + constructionCostsByProject.get(streetBaseDataContainer.getProjectInformation().getProjectNumber()) + )) + .toList(); + + logger.info( "Writing csv" ); + StreetCsvWriter csvWriter = new StreetCsvWriter( "output/street_data.csv" ); + Table table = csvWriter.writeCsv( allStreetBaseData ); + + // === + Figures1KN figures1 = new Figures1KN( table ); + Figures2KN figures2 = new Figures2KN( table ); + + List
plots1 = new ArrayList<>(); + List
plots2 = new ArrayList<>(); + + + plots1.add( figures1.fzkmFromTtime_vs_fzkmOrig() ); + plots1.add( figures1.fzkmFromTtimeSum_vs_fzkmOrig() ); + + plots1.add( figures1.invcost_tud_vs_orig() ); + plots1.add( figures1.nkv_el03() ); + plots1.add( figures1.nkv_carbon700() ); + plots1.add( figures1.nkv_el03_carbon700() ); + +// plots1.add( figures1.elasticities() ); +// plots1.add( figures1.fzkmDiff() ); +// plots1.add( figures1.carbon() ); +// plots1.add( figures1.invcost() ); +// plots1.add( figures1.nkv_el03_diff() ); +// plots1.add( figures1.dtv() ); +// plots1.add( figures1.fzkmNew() ); + + plots2.add( figures2.nkvVsDtv() ); + + plots2.add( figures2.cost_VS_nkvOrig() ); + + plots2.add( figures2.costOrigVsCumulativeCostOrig() ); + + plots2.addAll( figures2.nkvElttimeCarbon215(5 ) ); + + + + + + plots2.add( figures2.invcosttud_vs_nkvEl03Cprice215Invcosttud( 5) ); + plots2.add( figures2.cumulativeCostTud_vs_nkvEl03Cprice215InvcostTud(5 ) ); + plots2.add( figures2.cumulativeCostTud_vs_nkvEl03Cprice215InvcostTud(Integer.MAX_VALUE ) ); + plots2.add( figures2.invcosttud_vs_nkvEl03Cprice215Invcosttud( Integer.MAX_VALUE) ); + + plots2.add( figures2.invcosttud_vs_nkvElttimeCarbon700Invcosttud(5) ); + plots2.add( figures2.invcost50_vs_NkvEl03Cprice700InvcostTud() ); + plots2.add( figures2.cumcost50_vs_nkvEl03Cprice700InvcostTud() ); + plots2.add( figures2.invcosttud_vs_nkvElttimeCarbon2000Invcosttud() ); + + plots2.add( figures2.carbon_vs_nkvEl03Cprice215Invcost50Capped5() ); + + + // === + + + + String page = MultiPlotUtils.pageTop() + System.lineSeparator(); + for( int ii=0; ii comparator = ( o1, o2 ) -> { + Einstufung p1 = Einstufung.valueOf( o1.getString( Headers.EINSTUFUNG ) ); + Einstufung p2 = Einstufung.valueOf( o2.getString( Headers.EINSTUFUNG ) ); + return p1.compareTo( p2 ); + }; + table = table.sortOn( comparator ); + NumberFormat format = NumberFormat.getCompactNumberInstance(); + format.setMaximumFractionDigits( 0 ); + table.numberColumn( Headers.CO2_COST_EL03 ).setPrintFormatter( format, "n/a" ); + +// Table table2 = table.where( table.numberColumn( Headers.NKV_EL03_CARBON215_INVCOSTTUD ).isLessThan( 1. ) ); +// +// System.out.println(BvwpUtils.SEPARATOR); +// System.out.println( table.summarize( Headers.NKV_ORIG, count ).by(Headers.EINSTUFUNG ).print() ); +// System.out.println( System.lineSeparator() + "Davon müssen folgende nachbewertet werden:"); +// System.out.println( table2.summarize( Headers.NKV_ORIG, count ).by(Headers.EINSTUFUNG ) ); +// +// System.out.println(BvwpUtils.SEPARATOR); +// System.out.println( table.summarize( Headers.INVCOST_ORIG, sum, mean, stdDev, min, max ).by(Headers.EINSTUFUNG ) ); +// System.out.println( System.lineSeparator() + "Davon müssen folgende nachbewertet werden:"); +// System.out.println( table2.summarize( Headers.INVCOST_ORIG, sum, mean, stdDev, min, max ).by(Headers.EINSTUFUNG ) ); +// +// System.out.println(BvwpUtils.SEPARATOR); +// System.out.println( table.summarize( Headers.CO2_COST_EL03, sum, mean, stdDev, min, max ).by(Headers.EINSTUFUNG ) ); +// System.out.println( System.lineSeparator() + "Davon müssen folgende nachbewertet werden:"); +// System.out.println( table2.summarize( Headers.CO2_COST_EL03, sum, mean, stdDev, min, max ).by(Headers.EINSTUFUNG ) ); + } + +} From 9324eaf87cbbf2d5f856e75e7674d0adb9267f2d Mon Sep 17 00:00:00 2001 From: Kai Nagel Date: Wed, 29 May 2024 07:37:47 +0200 Subject: [PATCH 2/5] delete --- .../bvwp/users/kn/RunLocalRailScrapingKN.java | 155 ------------------ 1 file changed, 155 deletions(-) delete mode 100644 src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java diff --git a/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java b/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java deleted file mode 100644 index e05973d..0000000 --- a/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java +++ /dev/null @@ -1,155 +0,0 @@ -package org.tub.vsp.bvwp.users.kn; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.tub.vsp.bvwp.BvwpUtils; -import org.tub.vsp.bvwp.data.Headers; -import org.tub.vsp.bvwp.data.container.analysis.StreetAnalysisDataContainer; -import org.tub.vsp.bvwp.data.type.Einstufung; -import org.tub.vsp.bvwp.io.StreetCsvWriter; -import org.tub.vsp.bvwp.plot.MultiPlotUtils; -import org.tub.vsp.bvwp.scraping.RailScraper; -import org.tub.vsp.bvwp.scraping.Scraper; -import org.tub.vsp.bvwp.scraping.StreetScraper; -import tech.tablesaw.api.Row; -import tech.tablesaw.api.Table; -import tech.tablesaw.plotly.components.Figure; -import tech.tablesaw.plotly.display.Browser; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Paths; -import java.text.NumberFormat; -import java.util.*; - -public class RunLocalRailScrapingKN{ - private static final Logger logger = LogManager.getLogger( RunLocalRailScrapingKN.class ); - - public static void main(String[] args) throws IOException{ - Locale.setDefault( Locale.US ); - - RailScraper scraper = new RailScraper(); - - logger.info( "Starting scraping" ); - - String filePath = "../../shared-svn/"; - Map constructionCostsByProject = BvwpUtils.getConstructionCostsFromTudFile(filePath ); - - - // yyyy man könnte (sollte?) den table in den StreetAnalysisDataContainer mit hinein geben, und die Werte gleich dort eintragen. kai, feb'24 - List allStreetBaseData = scraper - .stream() - .map(streetBaseDataContainer -> new StreetAnalysisDataContainer( - streetBaseDataContainer, - constructionCostsByProject.get(streetBaseDataContainer.getProjectInformation().getProjectNumber()) - )) - .toList(); - - logger.info( "Writing csv" ); - StreetCsvWriter csvWriter = new StreetCsvWriter( "output/street_data.csv" ); - Table table = csvWriter.writeCsv( allStreetBaseData ); - - // === - Figures1KN figures1 = new Figures1KN( table ); - Figures2KN figures2 = new Figures2KN( table ); - - List
plots1 = new ArrayList<>(); - List
plots2 = new ArrayList<>(); - - - plots1.add( figures1.fzkmFromTtime_vs_fzkmOrig() ); - plots1.add( figures1.fzkmFromTtimeSum_vs_fzkmOrig() ); - - plots1.add( figures1.invcost_tud_vs_orig() ); - plots1.add( figures1.nkv_el03() ); - plots1.add( figures1.nkv_carbon700() ); - plots1.add( figures1.nkv_el03_carbon700() ); - -// plots1.add( figures1.elasticities() ); -// plots1.add( figures1.fzkmDiff() ); -// plots1.add( figures1.carbon() ); -// plots1.add( figures1.invcost() ); -// plots1.add( figures1.nkv_el03_diff() ); -// plots1.add( figures1.dtv() ); -// plots1.add( figures1.fzkmNew() ); - - plots2.add( figures2.nkvVsDtv() ); - - plots2.add( figures2.cost_VS_nkvOrig() ); - - plots2.add( figures2.costOrigVsCumulativeCostOrig() ); - - plots2.addAll( figures2.nkvElttimeCarbon215(5 ) ); - - - - - - plots2.add( figures2.invcosttud_vs_nkvEl03Cprice215Invcosttud( 5) ); - plots2.add( figures2.cumulativeCostTud_vs_nkvEl03Cprice215InvcostTud(5 ) ); - plots2.add( figures2.cumulativeCostTud_vs_nkvEl03Cprice215InvcostTud(Integer.MAX_VALUE ) ); - plots2.add( figures2.invcosttud_vs_nkvEl03Cprice215Invcosttud( Integer.MAX_VALUE) ); - - plots2.add( figures2.invcosttud_vs_nkvElttimeCarbon700Invcosttud(5) ); - plots2.add( figures2.invcost50_vs_NkvEl03Cprice700InvcostTud() ); - plots2.add( figures2.cumcost50_vs_nkvEl03Cprice700InvcostTud() ); - plots2.add( figures2.invcosttud_vs_nkvElttimeCarbon2000Invcosttud() ); - - plots2.add( figures2.carbon_vs_nkvEl03Cprice215Invcost50Capped5() ); - - - // === - - - - String page = MultiPlotUtils.pageTop() + System.lineSeparator(); - for( int ii=0; ii comparator = ( o1, o2 ) -> { - Einstufung p1 = Einstufung.valueOf( o1.getString( Headers.EINSTUFUNG ) ); - Einstufung p2 = Einstufung.valueOf( o2.getString( Headers.EINSTUFUNG ) ); - return p1.compareTo( p2 ); - }; - table = table.sortOn( comparator ); - NumberFormat format = NumberFormat.getCompactNumberInstance(); - format.setMaximumFractionDigits( 0 ); - table.numberColumn( Headers.CO2_COST_EL03 ).setPrintFormatter( format, "n/a" ); - -// Table table2 = table.where( table.numberColumn( Headers.NKV_EL03_CARBON215_INVCOSTTUD ).isLessThan( 1. ) ); -// -// System.out.println(BvwpUtils.SEPARATOR); -// System.out.println( table.summarize( Headers.NKV_ORIG, count ).by(Headers.EINSTUFUNG ).print() ); -// System.out.println( System.lineSeparator() + "Davon müssen folgende nachbewertet werden:"); -// System.out.println( table2.summarize( Headers.NKV_ORIG, count ).by(Headers.EINSTUFUNG ) ); -// -// System.out.println(BvwpUtils.SEPARATOR); -// System.out.println( table.summarize( Headers.INVCOST_ORIG, sum, mean, stdDev, min, max ).by(Headers.EINSTUFUNG ) ); -// System.out.println( System.lineSeparator() + "Davon müssen folgende nachbewertet werden:"); -// System.out.println( table2.summarize( Headers.INVCOST_ORIG, sum, mean, stdDev, min, max ).by(Headers.EINSTUFUNG ) ); -// -// System.out.println(BvwpUtils.SEPARATOR); -// System.out.println( table.summarize( Headers.CO2_COST_EL03, sum, mean, stdDev, min, max ).by(Headers.EINSTUFUNG ) ); -// System.out.println( System.lineSeparator() + "Davon müssen folgende nachbewertet werden:"); -// System.out.println( table2.summarize( Headers.CO2_COST_EL03, sum, mean, stdDev, min, max ).by(Headers.EINSTUFUNG ) ); - } - -} From f93b9ffad71276ac9abc922f2417b68846d7fc38 Mon Sep 17 00:00:00 2001 From: Kai Nagel Date: Wed, 29 May 2024 17:46:22 +0200 Subject: [PATCH 3/5] towards rail --- .../tub/vsp/bvwp/RunLocalRailScraping.java | 2 +- .../vsp/bvwp/computation/NkvCalculator.java | 2 + .../bvwp/computation/NkvCalculatorRail.java | 114 +++++++ .../analysis/RailAnalysisDataContainer.java | 90 +++++- .../analysis/StreetAnalysisDataContainer.java | 3 +- .../base/rail/RailBaseDataContainer.java | 6 +- ... => RailPhysicalEffectsDataContainer.java} | 6 +- .../RailPhysicalEffectMapper.java | 8 +- .../org/tub/vsp/bvwp/io/RailCsvWriter.java | 290 ++++++++++++++++++ .../bvwp/users/kn/RunLocalRailScrapingKN.java | 30 ++ 10 files changed, 538 insertions(+), 13 deletions(-) create mode 100644 src/main/java/org/tub/vsp/bvwp/computation/NkvCalculatorRail.java rename src/main/java/org/tub/vsp/bvwp/data/container/base/rail/{RailPhysicalEffectDataContainer.java => RailPhysicalEffectsDataContainer.java} (75%) create mode 100644 src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java diff --git a/src/main/java/org/tub/vsp/bvwp/RunLocalRailScraping.java b/src/main/java/org/tub/vsp/bvwp/RunLocalRailScraping.java index bac6472..6fea2b1 100644 --- a/src/main/java/org/tub/vsp/bvwp/RunLocalRailScraping.java +++ b/src/main/java/org/tub/vsp/bvwp/RunLocalRailScraping.java @@ -21,7 +21,7 @@ public static void main(String[] args) { .toList(); logger.info("Writing csv"); - RailCsvWriter csvWriter = new RailCsvWriter(); + RailCsvWriter csvWriter = new RailCsvWriter(""); //TODO } diff --git a/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculator.java b/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculator.java index ab51bd4..8c24a4d 100644 --- a/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculator.java +++ b/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculator.java @@ -29,6 +29,7 @@ public Double calculateCost_CO2( Modifications modifications ) { } // old static methods beyond: + @Deprecated // use instance approach public static Double calculateNkv(Modifications modifications, StreetBaseDataContainer streetBaseDataContainer) { // log.warn("modifications=" + modifications); Optional a = amountsFromStreetBaseData(streetBaseDataContainer); @@ -93,6 +94,7 @@ private static Optional benefitsFromStreetBaseData(StreetBaseDataConta // @formatter:on } + @Deprecated // use instance approach public static Double calculateCost_CO2( Modifications modifications, StreetBaseDataContainer streetBaseDataContainer ) { log.warn("modifications=" + modifications); Optional a = amountsFromStreetBaseData(streetBaseDataContainer); diff --git a/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculatorRail.java b/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculatorRail.java new file mode 100644 index 0000000..8f48058 --- /dev/null +++ b/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculatorRail.java @@ -0,0 +1,114 @@ +package org.tub.vsp.bvwp.computation; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.tub.vsp.bvwp.data.container.base.rail.RailBaseDataContainer; +import org.tub.vsp.bvwp.data.container.base.street.StreetBaseDataContainer; +import org.tub.vsp.bvwp.data.container.base.street.StreetPhysicalEffectDataContainer; +import org.tub.vsp.bvwp.data.type.Emission; +import org.tub.vsp.bvwp.data.type.VehicleEmissions; + +import java.util.Optional; + +import static org.tub.vsp.bvwp.computation.ComputationKN.*; +import static org.tub.vsp.bvwp.computation.ConsoleColors.TEXT_BLACK; +import static org.tub.vsp.bvwp.computation.ConsoleColors.TEXT_RED; + +public class NkvCalculatorRail{ + + private static final Logger log = LogManager.getLogger( NkvCalculatorRail.class ); + private final RailBaseDataContainer railBaseDataContainer; + + // moving towards replacing the stateless static functions by a modifiable instance approach: + public NkvCalculatorRail( RailBaseDataContainer streetBaseDataContainer ) { + this.railBaseDataContainer = streetBaseDataContainer; + } + public Double calculateNkv( Modifications modifications ) { + return calculateNkv( modifications, this.railBaseDataContainer ); + } + public Double calculateCost_CO2( Modifications modifications ) { + return calculateCost_CO2( modifications, this.railBaseDataContainer ); + } + + // old static methods beyond: + @Deprecated // use instance approach + public static Double calculateNkv( Modifications modifications, RailBaseDataContainer streetBaseDataContainer ) { +// log.warn("modifications=" + modifications); + Optional a = amountsFromBaseData(streetBaseDataContainer ); + Optional b = benefitsFromBaseData(streetBaseDataContainer ); + + if (a.isEmpty()) { + log.warn("amounts container is empty for project=" + streetBaseDataContainer.getUrl()); + return null; + } + if (b.isEmpty()) { + log.warn("benefits container is empty for project=" + streetBaseDataContainer.getUrl()); + return null; + } + + double baukosten = streetBaseDataContainer.getCostBenefitAnalysis().getCost().overallCosts() * modifications.constructionCostFactor(); + + return nkvOhneKR_induz(modifications, a.get(), b.get(), baukosten, streetBaseDataContainer.getCostBenefitAnalysis().getOverallBenefit().overall()); + } + + + private static Optional amountsFromBaseData( RailBaseDataContainer baseDataContainer ) { + + StreetPhysicalEffectDataContainer.Effect tt = baseDataContainer.getPhysicalEffect().getTravelTimes(); + + StreetPhysicalEffectDataContainer.Effect vkm = baseDataContainer.getPhysicalEffect() + .getVehicleKilometers(); + + VehicleEmissions vehicleEmissions = baseDataContainer.getPhysicalEffect().getEmissionsDataContainer() + .emissions().get(Emission.CO2); + + if (tt == null || vkm == null || vehicleEmissions == null) { + return Optional.empty(); + } + +// new Amounts( 1., 1., 1., 1., 1., 1., 1., 1. ); + // uncomment to see argument names + + final Amounts amounts = new Amounts( + vkm.overall(), Optional.ofNullable(vkm.induced()).orElse(0.), vkm.shifted(), // pkwkm + tt.overall(), Optional.ofNullable(tt.induced()).orElse(0.), tt.shifted(), // pers_h + vehicleEmissions.pkw(), vehicleEmissions.kfz() // co2 + ); + final Optional optional = Optional.of(amounts); + if (optional.isEmpty()) { + log.warn("here"); + throw new RuntimeException("stop"); + } + return optional; + } + + private static Optional benefitsFromBaseData( RailBaseDataContainer streetBaseDataContainer ) { + // @formatter:off + return Optional.ofNullable(streetBaseDataContainer).map(RailBaseDataContainer::getCostBenefitAnalysis) + .map(cb -> new Benefits( + cb.getNbOperations().overall(), // fzkm + cb.getNrz().overall(), // rz + cb.getNi().overall(), // impl + cb.getNl().overall(), // co2_infra + cb.getNa().get(Emission.CO2).overall(), // co2_betrieb + cb.getOverallBenefit().overall() // benefit + )); + // @formatter:on + } + + @Deprecated // use instance approach + public static Double calculateCost_CO2( Modifications modifications, RailBaseDataContainer streetBaseDataContainer ) { + log.warn("modifications=" + modifications); + Optional a = amountsFromBaseData(streetBaseDataContainer ); + Optional b = benefitsFromBaseData(streetBaseDataContainer ); + + if (a.isEmpty() || b.isEmpty()) { + throw new RuntimeException( "co2 costs cannot be computed" ); + } + final double co2Costs = -b_co2( modifications, a.get(), b.get() ); + + log.warn( TEXT_RED + "project=" + streetBaseDataContainer.getProjectInformation().getProjectNumber() + "; co2Costs=" + co2Costs + TEXT_BLACK ); + + return co2Costs; + } +} diff --git a/src/main/java/org/tub/vsp/bvwp/data/container/analysis/RailAnalysisDataContainer.java b/src/main/java/org/tub/vsp/bvwp/data/container/analysis/RailAnalysisDataContainer.java index 84e79ae..92643a7 100644 --- a/src/main/java/org/tub/vsp/bvwp/data/container/analysis/RailAnalysisDataContainer.java +++ b/src/main/java/org/tub/vsp/bvwp/data/container/analysis/RailAnalysisDataContainer.java @@ -2,16 +2,104 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.tub.vsp.bvwp.computation.ComputationKN; +import org.tub.vsp.bvwp.computation.Modifications; +import org.tub.vsp.bvwp.computation.NkvCalculatorRail; +import org.tub.vsp.bvwp.data.Headers; import org.tub.vsp.bvwp.data.container.base.rail.RailBaseDataContainer; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.SequencedMap; + +import static org.tub.vsp.bvwp.computation.Modifications.*; +import static org.tub.vsp.bvwp.data.container.analysis.StreetAnalysisDataContainer.INFLATION_Factor2022to2012; +import static org.tub.vsp.bvwp.data.type.Bautyp.*; + + public class RailAnalysisDataContainer { Logger logger = LogManager.getLogger(RailAnalysisDataContainer.class); private final RailBaseDataContainer baseDataContainer; - + private double constructionCostFactor = 1; + private final SequencedMap entries = new LinkedHashMap<>(); + private final List remarks = new ArrayList<>(); +// private final double constructionCostTud; public RailAnalysisDataContainer(RailBaseDataContainer baseDataContainer) { this.baseDataContainer = baseDataContainer; + this.addComputations(); } //add analysis stuff here... + private void addComputations() { + + double additionalLaneKm = baseDataContainer.getProjectInformation().getLength() * 2; + // (assumption 2 more lanes) + +// switch( baseDataContainer.getProjectInformation().getBautyp() ){ +// case NB4 -> additionalLaneKm *= 2; +// case NB6 -> additionalLaneKm *= 3; +// case NB4_EW4 -> additionalLaneKm *= 1.5; +// } + +// entries.put(Headers.VERKEHRSBELASTUNG_PLANFALL, baseDataContainer.getProjectInformation().getVerkehrsbelastungPlanfall() ); + + if ( additionalLaneKm==0. ) { + additionalLaneKm = 1.; // Knotenpunkt-Projekte; so that it becomes visible on logplot. kai, mar'24 + } + + entries.put( Headers.ADDTL_LANE_KM, additionalLaneKm ); + + double addtlFzkmFromElasticity03 = additionalLaneKm / ComputationKN.LANE_KM_AB * 0.3 * ComputationKN.FZKM_AB; +// final double addtlFzkmBeyondPrinsEl03 = addtlFzkmFromElasticity03 - baseDataContainer.getPhysicalEffect().getVehicleKilometers().overall(); + // (this is formulated such that addtlFzkmBeyondPrinsEl03=0 means the original additional Fzkm) + + entries.put(Headers.B_PER_KM, baseDataContainer.getCostBenefitAnalysis().getOverallBenefit().overall() / baseDataContainer.getProjectInformation().getLength() ); + + entries.put(Headers.NKV_ORIG, NkvCalculatorRail.calculateNkv( NO_CHANGE, baseDataContainer ) ); + entries.put(Headers.NKV_CO2, NkvCalculatorRail.calculateNkv( new Modifications( co2Price700, 0., 1, 1.75 ), baseDataContainer ) ); + entries.put(Headers.NKV_CO2_700_EN, NkvCalculatorRail.calculateNkv( new Modifications( co2Price700, 0., 1, 1.75 ), baseDataContainer ) ); + entries.put(Headers.NKV_CARBON700, NkvCalculatorRail.calculateNkv( new Modifications( co2Price700, 0., 1, 1.75 ), baseDataContainer ) ); + entries.put(Headers.NKV_CO2_2000_EN, NkvCalculatorRail.calculateNkv( new Modifications( 2000 * INFLATION_Factor2022to2012, 0, 1, 1. ), baseDataContainer ) ); +// entries.put(Headers.NKV_EL03, NkvCalculatorRail.calculateNkv( new Modifications( co2PriceBVWP, addtlFzkmBeyondPrinsEl03, 1, 1. ), baseDataContainer ) ); +// entries.put(Headers.NKV_EL03_CARBON215_INVCOSTTUD, NkvCalculatorRail.calculateNkv( new Modifications( co2Price215, addtlFzkmBeyondPrinsEl03, constructionCostFactor, 1. ), baseDataContainer ) ); +// entries.put(Headers.NKV_EL03_CARBON700tpr0_INVCOSTTUD, NkvCalculatorRail.calculateNkv( new Modifications( co2Price700, addtlFzkmBeyondPrinsEl03, constructionCostFactor, 1.75 ), baseDataContainer ) ); +// entries.put(Headers.NKV_EL03_CARBON700tpr0, NkvCalculatorRail.calculateNkv( new Modifications( co2Price700, addtlFzkmBeyondPrinsEl03, 1., 1.75 ), baseDataContainer ) ); +// entries.put(Headers.NKV_EL03_CO2_INVCOST50, NkvCalculatorRail.calculateNkv( new Modifications( co2Price700, addtlFzkmBeyondPrinsEl03, constructionCostFactor ), baseDataContainer ) ); + + entries.put(Headers.ADDTL_PKWKM_EL03, addtlFzkmFromElasticity03 ); + entries.put(Headers.CO2_COST_ORIG, Math.max( 1., NkvCalculatorRail.calculateCost_CO2( NO_CHANGE, baseDataContainer ) ) ); +// entries.put(Headers.CO2_COST_EL03, Math.max( 1., NkvCalculatorRail.calculateCost_CO2( new Modifications( co2PriceBVWP, addtlFzkmBeyondPrinsEl03, 1, 1. ), baseDataContainer ) ) ); + // ("max(1,...)" so that they become visible on logplot. find other solution! +// entries.put(Headers.INVCOST_TUD, this.constructionCostTud ); + + double AVERAGE_SPEED_OF_ADDITIONAL_TRAVEL = 50; // km/h +// double addtlFzkmFromTtime = - baseDataContainer.getPhysicalEffect().getVehicleHours().overall() * AVERAGE_SPEED_OF_ADDITIONAL_TRAVEL; +// entries.put( Headers.ADDTL_PKWKM_FROM_TTIME, addtlFzkmFromTtime ); + +// entries.put( Headers.NKV_ELTTIME_CARBON215_INVCOSTTUD, NkvCalculatorRail.calculateNkv( new Modifications( co2Price215, addtlFzkmFromTtime, constructionCostFactor, 1. ), baseDataContainer ) ); +// entries.put( Headers.NKV_ELTTIME_CARBON700TPR0_INVCOSTTUD, NkvCalculatorRail.calculateNkv( new Modifications( co2Price700, addtlFzkmFromTtime, constructionCostFactor, 1.75 ), baseDataContainer ) ); +// +// entries.put( Headers.NKV_ELTTIME_CARBON2000_INVCOSTTUD, NkvCalculatorRail.calculateNkv( new Modifications( 2000 * INFLATION_Factor2022to2012, addtlFzkmFromTtime, constructionCostFactor, 1.75 ), baseDataContainer ) ); + + if ( baseDataContainer.getProjectInformation().getProjectNumber().contains("A1-G50-NI" )) { + this.remarks.add("Eher geringer Benefit pro km ... erzeugt dann ueber die El pro km relativ viel Verkehr " + + "der per co2 stark negativ bewertet wird."); + } + + } + + public RailBaseDataContainer getBaseDataContainer() { + return baseDataContainer; + } + + public SequencedMap getColumns() { + return entries; + } + + public List getRemarks() { + return remarks; + } + } diff --git a/src/main/java/org/tub/vsp/bvwp/data/container/analysis/StreetAnalysisDataContainer.java b/src/main/java/org/tub/vsp/bvwp/data/container/analysis/StreetAnalysisDataContainer.java index d50b2e7..e062503 100644 --- a/src/main/java/org/tub/vsp/bvwp/data/container/analysis/StreetAnalysisDataContainer.java +++ b/src/main/java/org/tub/vsp/bvwp/data/container/analysis/StreetAnalysisDataContainer.java @@ -22,6 +22,7 @@ public class StreetAnalysisDataContainer { private final SequencedMap entries = new LinkedHashMap<>(); private final List remarks = new ArrayList<>(); private final double constructionCostTud; + static final double INFLATION_Factor2022to2012 = 0.917; // Zinse Wert von 2020 auf BVWP Zeitpunkt 2012 ab. public StreetAnalysisDataContainer(StreetBaseDataContainer streetBaseDataContainer, double investmentCostNew ) { this.streetBaseData = streetBaseDataContainer; @@ -56,8 +57,6 @@ private void addComputations() { case NB4_EW4 -> additionalLaneKm *= 1.5; } - final double INFLATION_Factor2022to2012 = 0.917; // Zinse Wert von 2020 auf BVWP Zeitpunkt 2012 ab. - entries.put(Headers.VERKEHRSBELASTUNG_PLANFALL, streetBaseData.getProjectInformation().getVerkehrsbelastungPlanfall() ); if ( additionalLaneKm==0. ) { diff --git a/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailBaseDataContainer.java b/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailBaseDataContainer.java index 6afa998..b5a2b83 100644 --- a/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailBaseDataContainer.java +++ b/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailBaseDataContainer.java @@ -6,7 +6,7 @@ public class RailBaseDataContainer { private String url; RailProjectInformationDataContainer projectInformation; - RailPhysicalEffectDataContainer physicalEffect; + RailPhysicalEffectsDataContainer physicalEffect; RailCostBenefitAnalysisDataContainer costBenefitAnalysis; public String getUrl() { @@ -27,11 +27,11 @@ public RailBaseDataContainer setProjectInformation(RailProjectInformationDataCon return this; } - public RailPhysicalEffectDataContainer getPhysicalEffect() { + public RailPhysicalEffectsDataContainer getPhysicalEffect() { return physicalEffect; } - public RailBaseDataContainer setPhysicalEffect(RailPhysicalEffectDataContainer physicalEffect) { + public RailBaseDataContainer setPhysicalEffect( RailPhysicalEffectsDataContainer physicalEffect ) { this.physicalEffect = physicalEffect; return this; } diff --git a/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailPhysicalEffectDataContainer.java b/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailPhysicalEffectsDataContainer.java similarity index 75% rename from src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailPhysicalEffectDataContainer.java rename to src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailPhysicalEffectsDataContainer.java index 396999d..24cc8fe 100644 --- a/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailPhysicalEffectDataContainer.java +++ b/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailPhysicalEffectsDataContainer.java @@ -2,14 +2,14 @@ import java.util.Objects; -public class RailPhysicalEffectDataContainer { +public class RailPhysicalEffectsDataContainer{ private RailEmissionsDataContainer emissionsDataContainer; public RailEmissionsDataContainer getEmissionsDataContainer() { return emissionsDataContainer; } - public RailPhysicalEffectDataContainer setEmissionsDataContainer(RailEmissionsDataContainer emissionsDataContainer) { + public RailPhysicalEffectsDataContainer setEmissionsDataContainer( RailEmissionsDataContainer emissionsDataContainer ) { this.emissionsDataContainer = emissionsDataContainer; return this; } @@ -23,7 +23,7 @@ public boolean equals(Object o) { return false; } - RailPhysicalEffectDataContainer that = (RailPhysicalEffectDataContainer) o; + RailPhysicalEffectsDataContainer that = (RailPhysicalEffectsDataContainer) o; return Objects.equals(emissionsDataContainer, that.emissionsDataContainer); } diff --git a/src/main/java/org/tub/vsp/bvwp/data/mapper/physicalEffect/RailPhysicalEffectMapper.java b/src/main/java/org/tub/vsp/bvwp/data/mapper/physicalEffect/RailPhysicalEffectMapper.java index 229e240..ac8b2e0 100644 --- a/src/main/java/org/tub/vsp/bvwp/data/mapper/physicalEffect/RailPhysicalEffectMapper.java +++ b/src/main/java/org/tub/vsp/bvwp/data/mapper/physicalEffect/RailPhysicalEffectMapper.java @@ -3,13 +3,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; -import org.tub.vsp.bvwp.data.container.base.rail.RailPhysicalEffectDataContainer; +import org.tub.vsp.bvwp.data.container.base.rail.RailPhysicalEffectsDataContainer; import org.tub.vsp.bvwp.data.mapper.physicalEffect.emissions.RailEmissionsMapper; public class RailPhysicalEffectMapper { private static final Logger logger = LogManager.getLogger(RailPhysicalEffectMapper.class); - public static RailPhysicalEffectDataContainer mapDocument(Document document) { - return new RailPhysicalEffectDataContainer().setEmissionsDataContainer(RailEmissionsMapper.mapDocument(document)); + public static RailPhysicalEffectsDataContainer mapDocument( Document document ) { + final RailPhysicalEffectsDataContainer physicalDataContainer = new RailPhysicalEffectsDataContainer(); + physicalDataContainer.setEmissionsDataContainer(RailEmissionsMapper.mapDocument(document ) ); + return physicalDataContainer; } } diff --git a/src/main/java/org/tub/vsp/bvwp/io/RailCsvWriter.java b/src/main/java/org/tub/vsp/bvwp/io/RailCsvWriter.java index 16625c0..8c4ba04 100644 --- a/src/main/java/org/tub/vsp/bvwp/io/RailCsvWriter.java +++ b/src/main/java/org/tub/vsp/bvwp/io/RailCsvWriter.java @@ -1,4 +1,294 @@ package org.tub.vsp.bvwp.io; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.tub.vsp.bvwp.data.Headers; +import org.tub.vsp.bvwp.data.container.analysis.RailAnalysisDataContainer; +import org.tub.vsp.bvwp.data.container.base.street.StreetBaseDataContainer; +import org.tub.vsp.bvwp.data.container.base.street.StreetCostBenefitAnalysisDataContainer; +import org.tub.vsp.bvwp.data.type.Benefit; +import org.tub.vsp.bvwp.data.type.Cost; +import org.tub.vsp.bvwp.data.type.Emission; +import tech.tablesaw.api.DoubleColumn; +import tech.tablesaw.api.Row; +import tech.tablesaw.api.StringColumn; +import tech.tablesaw.api.Table; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; public class RailCsvWriter { + private static final Logger logger = LogManager.getLogger(StreetCsvWriter.class); + private final String outputPath; + + private final Table table; + + public RailCsvWriter(String outputPath) { + this.outputPath = outputPath; + this.table = Table.create("defaultTable"); + // (I am somewhat abusing this class to also generate a TableSaw Table. Probably would make sense later to + // first genrate the Table object, + // and then csv-write that. kai, feb'24) + } + + public Table writeCsv( List analysisDataContainers ) { + logger.info("Writing csv."); + List headers = getHeaders(analysisDataContainers, table); + + //make sure that the directory exists + Path path = Paths.get(outputPath); + try { + Files.createDirectories(path.getParent()); + } catch (IOException e) { + logger.error("Could not create directory for file {}", path); + throw new RuntimeException(e); + } + + try ( + BufferedWriter writer = new BufferedWriter(new FileWriter(this.outputPath)); + CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.Builder.create() + .setNullString("") + .setHeader(headers.toArray(new String[0])) + .setDelimiter(';') + .build()) + ) { + for ( RailAnalysisDataContainer analysisDataContainer : analysisDataContainers) { + logger.info("Writing csv record for {}", analysisDataContainer.getBaseDataContainer().getUrl()); + final List csvRecord = getCsvRecord(analysisDataContainer, table); + csvPrinter.printRecord(csvRecord); + { + StringBuilder strb = new StringBuilder(); + for (int ii = 2; ii < csvRecord.size(); ii++) { + strb.append(csvRecord.get(ii)).append(";"); + } + System.out.println(strb); + } + } + csvPrinter.flush(); + logger.info("Finished writing csv."); + } catch (IOException e) { + throw new RuntimeException(e); + } + return table; + } + + private static class Record { + private final Row row; + private final List record = new ArrayList<>(); + private final Table table; + int ii = 0; + + private Record(Table table) { + this.row = table.appendRow(); + this.table = table; + } + + // private Record add( String str ) { +// row.setString( ii, str ); +// record.add( str ); +// ii++; +// return this; +// } + private Record add(String key, String str) { + +// if ( !table.columnNames().contains( key ) ) { +// table.addColumns( StringColumn.create( key ) ); +// } + // not working (I think) + + row.setString(key, str); + record.add(str); + ii++; + return this; + } + + // private Record add( Double dbl ) { +// if ( dbl != null ){ +// row.setDouble( ii, dbl ); +// } else { +// row.setDouble( ii, Double.NaN ); +// } +// record.add( dbl ); +// ii++; +// return this; +// } + private Record add(String key, Double dbl) { + +// if ( !table.columnNames().contains( key ) ) { +// table.addColumns( DoubleColumn.create( key ) ); +// } + // not working (I think) + + if (dbl != null) { + row.setDouble(key, dbl); + } else { + row.setDouble(key, Double.NaN); + } + record.add(dbl); + ii++; + return this; + } +// private Record addAll( Iterable objs ) { +// for( Object obj : objs ){ +// if ( obj==null ) { +// this.add( Double.NaN ); +// } else if ( obj instanceof String ) { +// this.add( (String) obj ); +// } else if ( obj instanceof Double ) { +// this.add( (Double) obj ); +// } else { +// throw new RuntimeException( "not implemented; obj=" + obj ); +// } +// } +// return this; +// } + } + + private static List getCsvRecord( RailAnalysisDataContainer analysisDataContainer, Table table ) { + StreetBaseDataContainer baseDataContainer = analysisDataContainer.getStreetBaseDataContainer(); + + Record record = new Record(table); + + //general info + record.add(Headers.PROJECT_NAME, baseDataContainer.getProjectInformation().getProjectNumber()); + record.add(Headers.LINK, baseDataContainer.getUrl()); + record.add(Headers.EINSTUFUNG, baseDataContainer.getProjectInformation().getPriority().name() ); + record.add(Headers.BAUTYP, baseDataContainer.getProjectInformation().getBautyp().name()); + + record.add(Headers.LENGTH, baseDataContainer.getProjectInformation().getLength()); + + record.add(Headers.ADDTL_PKWKM_ORIG, baseDataContainer.getPhysicalEffect().getVehicleKilometers().overall()); + record.add(Headers.ADDTL_PKWKM_INDUZ_ORIG, Optional.ofNullable(baseDataContainer.getPhysicalEffect().getVehicleKilometers().induced() ).orElse(0. ) ); + + record.add(Headers.B_PER_KM, baseDataContainer.getCostBenefitAnalysis().getNbOperations().overall()); + + //co2 equivalents + record.add(Headers.CO_2_EQUIVALENTS_EMISSIONS, baseDataContainer.getPhysicalEffect().getEmissionsDataContainer().co2Overall()); +// record.add(Headers.B_CO_2_EQUIVALENTS_ANNUAL, baseDataContainer.getCostBenefitAnalysis() +// .getCo2EquivalentBenefit() +// .annual()); // yyyy not needed + record.add(Headers.B_CO_2_EQUIVALENTS_ORIG, baseDataContainer.getCostBenefitAnalysis().getCo2EquivalentBenefit().overall()); + + //emissions +// for (Emission emission : EMISSION_COLUMNS.keySet()) { +// addEmissionsAnnualOverallBenefit(baseDataContainer, record, emission); +// } + + //overall benefit and cost + record.add(Headers.B_OVERALL, Optional.ofNullable(baseDataContainer.getCostBenefitAnalysis()) + .map(StreetCostBenefitAnalysisDataContainer::getOverallBenefit) + .map(Benefit::overall) + .orElse(null)); + record.add(Headers.INVCOST_ORIG, Optional.ofNullable(baseDataContainer.getCostBenefitAnalysis() ) + .map(StreetCostBenefitAnalysisDataContainer::getCost) + .map(Cost::overallCosts) + .orElse(null)); + // (yy warum diese aufwändige Syntax? kai, feb'24) + // --> da sowohl getCostBenefitAnalysis, getCost als auch overallCosts null zurückgeben können, wenn die + // Daten nicht vorhanden sind. So spart man sich null checks (paul, feb'24) + + for (Map.Entry entry : analysisDataContainer.getColumns().entrySet()) { + record.add(entry.getKey(), entry.getValue()); + } + + int ii = 0; + for (String remark : analysisDataContainer.getRemarks()) { + record.add("remark_" + ii, "\"" + remark + "\""); + ii++; + } + + return record.record; + } + + private static class HeadersAdder { + private final Table table; + private final List headers = new ArrayList<>(); + + private HeadersAdder(Table table) { + this.table = table; + } + + private HeadersAdder addStringColumn(String name) { + headers.add(name); + table.addColumns(StringColumn.create(name)); + return this; + } + + private HeadersAdder addDoubleColumn(String name) { + headers.add(name); + table.addColumns(DoubleColumn.create(name)); + return this; + } + } + + private static List getHeaders( List analysisDataContainers, Table table ) { + //assert that all entries of new nkv have the same keys + assert analysisDataContainers.stream() + .allMatch(a -> { + Set thisKeys = a.getColumns().keySet(); + Set firstKeys = analysisDataContainers.getFirst().getColumns() + .keySet(); + return thisKeys.containsAll(firstKeys) && thisKeys.size() == firstKeys.size(); + }) : "Not all nkv have the same keys"; + + HeadersAdder headers = new HeadersAdder(table); + headers.addStringColumn(Headers.PROJECT_NAME); + + headers.addStringColumn(Headers.LINK); + headers.addStringColumn( Headers.EINSTUFUNG ); + headers.addStringColumn(Headers.BAUTYP); + headers.addDoubleColumn(Headers.LENGTH); + + headers.addDoubleColumn( Headers.ADDTL_PKWKM_ORIG ); + headers.addDoubleColumn(Headers.ADDTL_PKWKM_INDUZ_ORIG ); +// headers.addDoubleColumn( Headers.PKWKM_INDUZ_NEU ); // added by automagic + headers.addDoubleColumn(Headers.B_FZKM); + + headers.addDoubleColumn(Headers.CO_2_EQUIVALENTS_EMISSIONS); +// headers.addDoubleColumn( Headers.B_CO_2_EQUIVALENTS_ANNUAL ); + headers.addDoubleColumn( Headers.B_CO_2_EQUIVALENTS_ORIG ); + +// for (String colName : EMISSION_COLUMNS.values()) { +// headers.add(colName + "-emissions"); +// headers.add(colName + "-annual"); +// headers.add(colName + "-overall"); +// } + + headers.addDoubleColumn(Headers.B_OVERALL); + headers.addDoubleColumn(Headers.INVCOST_ORIG ); + + for (String s : analysisDataContainers.getFirst() + .getColumns() + .keySet()) { + headers.addDoubleColumn(s); + } + + for (int ii = 0; ii < 5; ii++) { + headers.addStringColumn("remark_" + ii); + } + + return headers.headers; + } + + private static void addEmissionsAnnualOverallBenefit(StreetBaseDataContainer baseDataContainer, + List record, Emission emission) { + record.add(baseDataContainer.getPhysicalEffect() + .getKfzEmission(emission)); + record.add(Optional.ofNullable(baseDataContainer.getCostBenefitAnalysis()) + .map(StreetCostBenefitAnalysisDataContainer::getNa) + .map(m -> m.get(emission)) + .map(Benefit::annual) + .orElse(null)); + record.add(Optional.ofNullable(baseDataContainer.getCostBenefitAnalysis()) + .map(StreetCostBenefitAnalysisDataContainer::getNa) + .map(m -> m.get(emission)) + .map(Benefit::overall) + .orElse(null)); + } } diff --git a/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java b/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java new file mode 100644 index 0000000..ae31b5a --- /dev/null +++ b/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java @@ -0,0 +1,30 @@ +package org.tub.vsp.bvwp.users.kn; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.tub.vsp.bvwp.data.container.analysis.RailAnalysisDataContainer; +import org.tub.vsp.bvwp.io.RailCsvWriter; +import org.tub.vsp.bvwp.scraping.RailScraper; +import tech.tablesaw.api.Table; + +import java.util.List; + +public class RunLocalRailScrapingKN{ + private static final Logger logger = LogManager.getLogger( RunLocalRailScrapingKN.class ); + + public static void main(String[] args) { + RailScraper scraper = new RailScraper(); + + logger.info("Starting scraping"); + List allRailData = scraper.extractAllLocalBaseData("./data/rail/all", "", "^2.*", "") + .stream() + .map(RailAnalysisDataContainer::new) + .toList(); + + logger.info("Writing csv"); + RailCsvWriter csvWriter = new RailCsvWriter("output/rail_data.csv"); + Table table = csvWriter.writeCsv( allRailData ); + + //TODO + } +} From e85622cabbb234b61e7c99e19cc8647344ebc725 Mon Sep 17 00:00:00 2001 From: Paul Heinrich Date: Fri, 31 May 2024 13:26:00 +0200 Subject: [PATCH 4/5] add general rail computation infrastructure --- .../tub/vsp/bvwp/RunLocalRailScraping.java | 14 +- .../vsp/bvwp/computation/ComputationKN.java | 15 + .../vsp/bvwp/computation/Modifications.java | 2 + .../bvwp/computation/NkvCalculatorRail.java | 116 ++----- .../analysis/RailAnalysisDataContainer.java | 12 +- .../org/tub/vsp/bvwp/io/RailCsvWriter.java | 294 ------------------ .../org/tub/vsp/bvwp/io/RailTableCreator.java | 175 +++++++++++ .../tub/vsp/bvwp/scraping/RailScraper.java | 6 +- .../bvwp/users/kn/RunLocalRailScrapingKN.java | 7 +- 9 files changed, 241 insertions(+), 400 deletions(-) delete mode 100644 src/main/java/org/tub/vsp/bvwp/io/RailCsvWriter.java create mode 100644 src/main/java/org/tub/vsp/bvwp/io/RailTableCreator.java diff --git a/src/main/java/org/tub/vsp/bvwp/RunLocalRailScraping.java b/src/main/java/org/tub/vsp/bvwp/RunLocalRailScraping.java index 6fea2b1..58c7419 100644 --- a/src/main/java/org/tub/vsp/bvwp/RunLocalRailScraping.java +++ b/src/main/java/org/tub/vsp/bvwp/RunLocalRailScraping.java @@ -3,26 +3,30 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.tub.vsp.bvwp.data.container.analysis.RailAnalysisDataContainer; -import org.tub.vsp.bvwp.io.RailCsvWriter; +import org.tub.vsp.bvwp.io.RailTableCreator; import org.tub.vsp.bvwp.scraping.RailScraper; +import tech.tablesaw.api.Table; +import java.io.IOException; import java.util.List; public class RunLocalRailScraping { private static final Logger logger = LogManager.getLogger(RunLocalRailScraping.class); - public static void main(String[] args) { + public static void main(String[] args) throws IOException { RailScraper scraper = new RailScraper(); logger.info("Starting scraping"); List allRailData = scraper.extractAllLocalBaseData("./data/rail/all", "", "^2.*", "") .stream() + //there is something wrong with the project number of 2-009-V03 + .filter(d -> !d.getProjectInformation().getProjectNumber().equals("2-009-V03")) .map(RailAnalysisDataContainer::new) .toList(); logger.info("Writing csv"); - RailCsvWriter csvWriter = new RailCsvWriter(""); - - //TODO + RailTableCreator tableCreator = new RailTableCreator(); + Table table = tableCreator.computeTable(allRailData); + table.write().csv("out.csv"); } } diff --git a/src/main/java/org/tub/vsp/bvwp/computation/ComputationKN.java b/src/main/java/org/tub/vsp/bvwp/computation/ComputationKN.java index 450e471..c82fb46 100644 --- a/src/main/java/org/tub/vsp/bvwp/computation/ComputationKN.java +++ b/src/main/java/org/tub/vsp/bvwp/computation/ComputationKN.java @@ -288,4 +288,19 @@ static Double b_co2(Modifications modifications, Amounts amounts, Benefits benef // Note that this really says nothing about old vs new co2 price, or old vs new addl traffic. That all depends on the settings in "modifications". } + + /** + * @param co2_infra_eur benefit in €/a + * @param co2_betrieb_t emissions in t/a + * @return new nkv + */ + static double nkv_rail(double co2_price, double baukosten, double benefit, double co2_infra_eur, double co2_betrieb_t) { + benefit -= co2_infra_eur; + benefit -= co2_betrieb_t; + + benefit += co2_price / 145. * co2_infra_eur; + benefit += co2_price * co2_betrieb_t; + + return benefit / baukosten; + } } diff --git a/src/main/java/org/tub/vsp/bvwp/computation/Modifications.java b/src/main/java/org/tub/vsp/bvwp/computation/Modifications.java index f509042..f8938fc 100644 --- a/src/main/java/org/tub/vsp/bvwp/computation/Modifications.java +++ b/src/main/java/org/tub/vsp/bvwp/computation/Modifications.java @@ -4,6 +4,8 @@ import org.apache.logging.log4j.Logger; public record Modifications(double co2Price, double mehrFzkm, double constructionCostFactor, double nonCo2BenefitsFactor) { + //TODO nonCo2BenefitsFactor is not used in the codebase + private static final Logger log = LogManager.getLogger(Modifications.class); public static final double co2PriceBVWP = 145.; public static final double co2Price700 = 642.; diff --git a/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculatorRail.java b/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculatorRail.java index 8f48058..8e32096 100644 --- a/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculatorRail.java +++ b/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculatorRail.java @@ -3,16 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.tub.vsp.bvwp.data.container.base.rail.RailBaseDataContainer; -import org.tub.vsp.bvwp.data.container.base.street.StreetBaseDataContainer; -import org.tub.vsp.bvwp.data.container.base.street.StreetPhysicalEffectDataContainer; import org.tub.vsp.bvwp.data.type.Emission; -import org.tub.vsp.bvwp.data.type.VehicleEmissions; - -import java.util.Optional; - -import static org.tub.vsp.bvwp.computation.ComputationKN.*; -import static org.tub.vsp.bvwp.computation.ConsoleColors.TEXT_BLACK; -import static org.tub.vsp.bvwp.computation.ConsoleColors.TEXT_RED; public class NkvCalculatorRail{ @@ -20,95 +11,42 @@ public class NkvCalculatorRail{ private final RailBaseDataContainer railBaseDataContainer; // moving towards replacing the stateless static functions by a modifiable instance approach: - public NkvCalculatorRail( RailBaseDataContainer streetBaseDataContainer ) { - this.railBaseDataContainer = streetBaseDataContainer; + public NkvCalculatorRail(RailBaseDataContainer railBaseDataContainer) { + this.railBaseDataContainer = railBaseDataContainer; } public Double calculateNkv( Modifications modifications ) { return calculateNkv( modifications, this.railBaseDataContainer ); } - public Double calculateCost_CO2( Modifications modifications ) { - return calculateCost_CO2( modifications, this.railBaseDataContainer ); - } + + // public Double calculateCost_CO2( Modifications modifications ) { +// return calculateCost_CO2( modifications, this.railBaseDataContainer ); +// } // old static methods beyond: @Deprecated // use instance approach - public static Double calculateNkv( Modifications modifications, RailBaseDataContainer streetBaseDataContainer ) { -// log.warn("modifications=" + modifications); - Optional a = amountsFromBaseData(streetBaseDataContainer ); - Optional b = benefitsFromBaseData(streetBaseDataContainer ); - - if (a.isEmpty()) { - log.warn("amounts container is empty for project=" + streetBaseDataContainer.getUrl()); - return null; - } - if (b.isEmpty()) { - log.warn("benefits container is empty for project=" + streetBaseDataContainer.getUrl()); - return null; - } + public static Double calculateNkv(Modifications modifications, RailBaseDataContainer railBaseDataContainer) { + assert modifications.mehrFzkm() == 0; + assert modifications.nonCo2BenefitsFactor() == 1.; - double baukosten = streetBaseDataContainer.getCostBenefitAnalysis().getCost().overallCosts() * modifications.constructionCostFactor(); + double baukosten = railBaseDataContainer.getCostBenefitAnalysis().getCost().overallCosts() * modifications.constructionCostFactor(); + double benefit = railBaseDataContainer.getCostBenefitAnalysis().getOverallBenefit().overall(); + double co2_infra_eur = railBaseDataContainer.getCostBenefitAnalysis().getNl().overall(); + double co2_betrieb_t = railBaseDataContainer.getPhysicalEffect().getEmissionsDataContainer().emissions().get(Emission.CO2); - return nkvOhneKR_induz(modifications, a.get(), b.get(), baukosten, streetBaseDataContainer.getCostBenefitAnalysis().getOverallBenefit().overall()); + return ComputationKN.nkv_rail(modifications.co2Price(), baukosten, benefit, co2_infra_eur, co2_betrieb_t); } - - private static Optional amountsFromBaseData( RailBaseDataContainer baseDataContainer ) { - - StreetPhysicalEffectDataContainer.Effect tt = baseDataContainer.getPhysicalEffect().getTravelTimes(); - - StreetPhysicalEffectDataContainer.Effect vkm = baseDataContainer.getPhysicalEffect() - .getVehicleKilometers(); - - VehicleEmissions vehicleEmissions = baseDataContainer.getPhysicalEffect().getEmissionsDataContainer() - .emissions().get(Emission.CO2); - - if (tt == null || vkm == null || vehicleEmissions == null) { - return Optional.empty(); - } - -// new Amounts( 1., 1., 1., 1., 1., 1., 1., 1. ); - // uncomment to see argument names - - final Amounts amounts = new Amounts( - vkm.overall(), Optional.ofNullable(vkm.induced()).orElse(0.), vkm.shifted(), // pkwkm - tt.overall(), Optional.ofNullable(tt.induced()).orElse(0.), tt.shifted(), // pers_h - vehicleEmissions.pkw(), vehicleEmissions.kfz() // co2 - ); - final Optional optional = Optional.of(amounts); - if (optional.isEmpty()) { - log.warn("here"); - throw new RuntimeException("stop"); - } - return optional; - } - - private static Optional benefitsFromBaseData( RailBaseDataContainer streetBaseDataContainer ) { - // @formatter:off - return Optional.ofNullable(streetBaseDataContainer).map(RailBaseDataContainer::getCostBenefitAnalysis) - .map(cb -> new Benefits( - cb.getNbOperations().overall(), // fzkm - cb.getNrz().overall(), // rz - cb.getNi().overall(), // impl - cb.getNl().overall(), // co2_infra - cb.getNa().get(Emission.CO2).overall(), // co2_betrieb - cb.getOverallBenefit().overall() // benefit - )); - // @formatter:on - } - - @Deprecated // use instance approach - public static Double calculateCost_CO2( Modifications modifications, RailBaseDataContainer streetBaseDataContainer ) { - log.warn("modifications=" + modifications); - Optional a = amountsFromBaseData(streetBaseDataContainer ); - Optional b = benefitsFromBaseData(streetBaseDataContainer ); - - if (a.isEmpty() || b.isEmpty()) { - throw new RuntimeException( "co2 costs cannot be computed" ); - } - final double co2Costs = -b_co2( modifications, a.get(), b.get() ); - - log.warn( TEXT_RED + "project=" + streetBaseDataContainer.getProjectInformation().getProjectNumber() + "; co2Costs=" + co2Costs + TEXT_BLACK ); - - return co2Costs; - } +// @Deprecated // use instance approach +// public static Double calculateCost_CO2( Modifications modifications, RailBaseDataContainer streetBaseDataContainer ) { +// log.warn("modifications=" + modifications); +// Amounts a = new Amounts(); +// Benefits b = benefitsFromBaseData(streetBaseDataContainer ); +// +// final double co2Costs = -b_co2( modifications, a, b ); +// +// log.warn( TEXT_RED + "project=" + streetBaseDataContainer.getProjectInformation().getProjectNumber() + "; co2Costs=" + co2Costs + +// TEXT_BLACK ); +// +// return co2Costs; +// } } diff --git a/src/main/java/org/tub/vsp/bvwp/data/container/analysis/RailAnalysisDataContainer.java b/src/main/java/org/tub/vsp/bvwp/data/container/analysis/RailAnalysisDataContainer.java index 92643a7..5199d1c 100644 --- a/src/main/java/org/tub/vsp/bvwp/data/container/analysis/RailAnalysisDataContainer.java +++ b/src/main/java/org/tub/vsp/bvwp/data/container/analysis/RailAnalysisDataContainer.java @@ -13,9 +13,9 @@ import java.util.List; import java.util.SequencedMap; -import static org.tub.vsp.bvwp.computation.Modifications.*; +import static org.tub.vsp.bvwp.computation.Modifications.NO_CHANGE; +import static org.tub.vsp.bvwp.computation.Modifications.co2Price700; import static org.tub.vsp.bvwp.data.container.analysis.StreetAnalysisDataContainer.INFLATION_Factor2022to2012; -import static org.tub.vsp.bvwp.data.type.Bautyp.*; public class RailAnalysisDataContainer { @@ -58,9 +58,9 @@ private void addComputations() { entries.put(Headers.B_PER_KM, baseDataContainer.getCostBenefitAnalysis().getOverallBenefit().overall() / baseDataContainer.getProjectInformation().getLength() ); entries.put(Headers.NKV_ORIG, NkvCalculatorRail.calculateNkv( NO_CHANGE, baseDataContainer ) ); - entries.put(Headers.NKV_CO2, NkvCalculatorRail.calculateNkv( new Modifications( co2Price700, 0., 1, 1.75 ), baseDataContainer ) ); - entries.put(Headers.NKV_CO2_700_EN, NkvCalculatorRail.calculateNkv( new Modifications( co2Price700, 0., 1, 1.75 ), baseDataContainer ) ); - entries.put(Headers.NKV_CARBON700, NkvCalculatorRail.calculateNkv( new Modifications( co2Price700, 0., 1, 1.75 ), baseDataContainer ) ); + entries.put(Headers.NKV_CO2, NkvCalculatorRail.calculateNkv(new Modifications(co2Price700, 0., 1, 1.), baseDataContainer)); + entries.put(Headers.NKV_CO2_700_EN, NkvCalculatorRail.calculateNkv(new Modifications(co2Price700, 0., 1, 1.), baseDataContainer)); + entries.put(Headers.NKV_CARBON700, NkvCalculatorRail.calculateNkv(new Modifications(co2Price700, 0., 1, 1.), baseDataContainer)); entries.put(Headers.NKV_CO2_2000_EN, NkvCalculatorRail.calculateNkv( new Modifications( 2000 * INFLATION_Factor2022to2012, 0, 1, 1. ), baseDataContainer ) ); // entries.put(Headers.NKV_EL03, NkvCalculatorRail.calculateNkv( new Modifications( co2PriceBVWP, addtlFzkmBeyondPrinsEl03, 1, 1. ), baseDataContainer ) ); // entries.put(Headers.NKV_EL03_CARBON215_INVCOSTTUD, NkvCalculatorRail.calculateNkv( new Modifications( co2Price215, addtlFzkmBeyondPrinsEl03, constructionCostFactor, 1. ), baseDataContainer ) ); @@ -69,7 +69,7 @@ private void addComputations() { // entries.put(Headers.NKV_EL03_CO2_INVCOST50, NkvCalculatorRail.calculateNkv( new Modifications( co2Price700, addtlFzkmBeyondPrinsEl03, constructionCostFactor ), baseDataContainer ) ); entries.put(Headers.ADDTL_PKWKM_EL03, addtlFzkmFromElasticity03 ); - entries.put(Headers.CO2_COST_ORIG, Math.max( 1., NkvCalculatorRail.calculateCost_CO2( NO_CHANGE, baseDataContainer ) ) ); +// entries.put(Headers.CO2_COST_ORIG, Math.max( 1., NkvCalculatorRail.calculateCost_CO2( NO_CHANGE, baseDataContainer ) ) ); // entries.put(Headers.CO2_COST_EL03, Math.max( 1., NkvCalculatorRail.calculateCost_CO2( new Modifications( co2PriceBVWP, addtlFzkmBeyondPrinsEl03, 1, 1. ), baseDataContainer ) ) ); // ("max(1,...)" so that they become visible on logplot. find other solution! // entries.put(Headers.INVCOST_TUD, this.constructionCostTud ); diff --git a/src/main/java/org/tub/vsp/bvwp/io/RailCsvWriter.java b/src/main/java/org/tub/vsp/bvwp/io/RailCsvWriter.java deleted file mode 100644 index 8c4ba04..0000000 --- a/src/main/java/org/tub/vsp/bvwp/io/RailCsvWriter.java +++ /dev/null @@ -1,294 +0,0 @@ -package org.tub.vsp.bvwp.io; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.tub.vsp.bvwp.data.Headers; -import org.tub.vsp.bvwp.data.container.analysis.RailAnalysisDataContainer; -import org.tub.vsp.bvwp.data.container.base.street.StreetBaseDataContainer; -import org.tub.vsp.bvwp.data.container.base.street.StreetCostBenefitAnalysisDataContainer; -import org.tub.vsp.bvwp.data.type.Benefit; -import org.tub.vsp.bvwp.data.type.Cost; -import org.tub.vsp.bvwp.data.type.Emission; -import tech.tablesaw.api.DoubleColumn; -import tech.tablesaw.api.Row; -import tech.tablesaw.api.StringColumn; -import tech.tablesaw.api.Table; - -import java.io.BufferedWriter; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; -public class RailCsvWriter { - private static final Logger logger = LogManager.getLogger(StreetCsvWriter.class); - private final String outputPath; - - private final Table table; - - public RailCsvWriter(String outputPath) { - this.outputPath = outputPath; - this.table = Table.create("defaultTable"); - // (I am somewhat abusing this class to also generate a TableSaw Table. Probably would make sense later to - // first genrate the Table object, - // and then csv-write that. kai, feb'24) - } - - public Table writeCsv( List analysisDataContainers ) { - logger.info("Writing csv."); - List headers = getHeaders(analysisDataContainers, table); - - //make sure that the directory exists - Path path = Paths.get(outputPath); - try { - Files.createDirectories(path.getParent()); - } catch (IOException e) { - logger.error("Could not create directory for file {}", path); - throw new RuntimeException(e); - } - - try ( - BufferedWriter writer = new BufferedWriter(new FileWriter(this.outputPath)); - CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.Builder.create() - .setNullString("") - .setHeader(headers.toArray(new String[0])) - .setDelimiter(';') - .build()) - ) { - for ( RailAnalysisDataContainer analysisDataContainer : analysisDataContainers) { - logger.info("Writing csv record for {}", analysisDataContainer.getBaseDataContainer().getUrl()); - final List csvRecord = getCsvRecord(analysisDataContainer, table); - csvPrinter.printRecord(csvRecord); - { - StringBuilder strb = new StringBuilder(); - for (int ii = 2; ii < csvRecord.size(); ii++) { - strb.append(csvRecord.get(ii)).append(";"); - } - System.out.println(strb); - } - } - csvPrinter.flush(); - logger.info("Finished writing csv."); - } catch (IOException e) { - throw new RuntimeException(e); - } - return table; - } - - private static class Record { - private final Row row; - private final List record = new ArrayList<>(); - private final Table table; - int ii = 0; - - private Record(Table table) { - this.row = table.appendRow(); - this.table = table; - } - - // private Record add( String str ) { -// row.setString( ii, str ); -// record.add( str ); -// ii++; -// return this; -// } - private Record add(String key, String str) { - -// if ( !table.columnNames().contains( key ) ) { -// table.addColumns( StringColumn.create( key ) ); -// } - // not working (I think) - - row.setString(key, str); - record.add(str); - ii++; - return this; - } - - // private Record add( Double dbl ) { -// if ( dbl != null ){ -// row.setDouble( ii, dbl ); -// } else { -// row.setDouble( ii, Double.NaN ); -// } -// record.add( dbl ); -// ii++; -// return this; -// } - private Record add(String key, Double dbl) { - -// if ( !table.columnNames().contains( key ) ) { -// table.addColumns( DoubleColumn.create( key ) ); -// } - // not working (I think) - - if (dbl != null) { - row.setDouble(key, dbl); - } else { - row.setDouble(key, Double.NaN); - } - record.add(dbl); - ii++; - return this; - } -// private Record addAll( Iterable objs ) { -// for( Object obj : objs ){ -// if ( obj==null ) { -// this.add( Double.NaN ); -// } else if ( obj instanceof String ) { -// this.add( (String) obj ); -// } else if ( obj instanceof Double ) { -// this.add( (Double) obj ); -// } else { -// throw new RuntimeException( "not implemented; obj=" + obj ); -// } -// } -// return this; -// } - } - - private static List getCsvRecord( RailAnalysisDataContainer analysisDataContainer, Table table ) { - StreetBaseDataContainer baseDataContainer = analysisDataContainer.getStreetBaseDataContainer(); - - Record record = new Record(table); - - //general info - record.add(Headers.PROJECT_NAME, baseDataContainer.getProjectInformation().getProjectNumber()); - record.add(Headers.LINK, baseDataContainer.getUrl()); - record.add(Headers.EINSTUFUNG, baseDataContainer.getProjectInformation().getPriority().name() ); - record.add(Headers.BAUTYP, baseDataContainer.getProjectInformation().getBautyp().name()); - - record.add(Headers.LENGTH, baseDataContainer.getProjectInformation().getLength()); - - record.add(Headers.ADDTL_PKWKM_ORIG, baseDataContainer.getPhysicalEffect().getVehicleKilometers().overall()); - record.add(Headers.ADDTL_PKWKM_INDUZ_ORIG, Optional.ofNullable(baseDataContainer.getPhysicalEffect().getVehicleKilometers().induced() ).orElse(0. ) ); - - record.add(Headers.B_PER_KM, baseDataContainer.getCostBenefitAnalysis().getNbOperations().overall()); - - //co2 equivalents - record.add(Headers.CO_2_EQUIVALENTS_EMISSIONS, baseDataContainer.getPhysicalEffect().getEmissionsDataContainer().co2Overall()); -// record.add(Headers.B_CO_2_EQUIVALENTS_ANNUAL, baseDataContainer.getCostBenefitAnalysis() -// .getCo2EquivalentBenefit() -// .annual()); // yyyy not needed - record.add(Headers.B_CO_2_EQUIVALENTS_ORIG, baseDataContainer.getCostBenefitAnalysis().getCo2EquivalentBenefit().overall()); - - //emissions -// for (Emission emission : EMISSION_COLUMNS.keySet()) { -// addEmissionsAnnualOverallBenefit(baseDataContainer, record, emission); -// } - - //overall benefit and cost - record.add(Headers.B_OVERALL, Optional.ofNullable(baseDataContainer.getCostBenefitAnalysis()) - .map(StreetCostBenefitAnalysisDataContainer::getOverallBenefit) - .map(Benefit::overall) - .orElse(null)); - record.add(Headers.INVCOST_ORIG, Optional.ofNullable(baseDataContainer.getCostBenefitAnalysis() ) - .map(StreetCostBenefitAnalysisDataContainer::getCost) - .map(Cost::overallCosts) - .orElse(null)); - // (yy warum diese aufwändige Syntax? kai, feb'24) - // --> da sowohl getCostBenefitAnalysis, getCost als auch overallCosts null zurückgeben können, wenn die - // Daten nicht vorhanden sind. So spart man sich null checks (paul, feb'24) - - for (Map.Entry entry : analysisDataContainer.getColumns().entrySet()) { - record.add(entry.getKey(), entry.getValue()); - } - - int ii = 0; - for (String remark : analysisDataContainer.getRemarks()) { - record.add("remark_" + ii, "\"" + remark + "\""); - ii++; - } - - return record.record; - } - - private static class HeadersAdder { - private final Table table; - private final List headers = new ArrayList<>(); - - private HeadersAdder(Table table) { - this.table = table; - } - - private HeadersAdder addStringColumn(String name) { - headers.add(name); - table.addColumns(StringColumn.create(name)); - return this; - } - - private HeadersAdder addDoubleColumn(String name) { - headers.add(name); - table.addColumns(DoubleColumn.create(name)); - return this; - } - } - - private static List getHeaders( List analysisDataContainers, Table table ) { - //assert that all entries of new nkv have the same keys - assert analysisDataContainers.stream() - .allMatch(a -> { - Set thisKeys = a.getColumns().keySet(); - Set firstKeys = analysisDataContainers.getFirst().getColumns() - .keySet(); - return thisKeys.containsAll(firstKeys) && thisKeys.size() == firstKeys.size(); - }) : "Not all nkv have the same keys"; - - HeadersAdder headers = new HeadersAdder(table); - headers.addStringColumn(Headers.PROJECT_NAME); - - headers.addStringColumn(Headers.LINK); - headers.addStringColumn( Headers.EINSTUFUNG ); - headers.addStringColumn(Headers.BAUTYP); - headers.addDoubleColumn(Headers.LENGTH); - - headers.addDoubleColumn( Headers.ADDTL_PKWKM_ORIG ); - headers.addDoubleColumn(Headers.ADDTL_PKWKM_INDUZ_ORIG ); -// headers.addDoubleColumn( Headers.PKWKM_INDUZ_NEU ); // added by automagic - headers.addDoubleColumn(Headers.B_FZKM); - - headers.addDoubleColumn(Headers.CO_2_EQUIVALENTS_EMISSIONS); -// headers.addDoubleColumn( Headers.B_CO_2_EQUIVALENTS_ANNUAL ); - headers.addDoubleColumn( Headers.B_CO_2_EQUIVALENTS_ORIG ); - -// for (String colName : EMISSION_COLUMNS.values()) { -// headers.add(colName + "-emissions"); -// headers.add(colName + "-annual"); -// headers.add(colName + "-overall"); -// } - - headers.addDoubleColumn(Headers.B_OVERALL); - headers.addDoubleColumn(Headers.INVCOST_ORIG ); - - for (String s : analysisDataContainers.getFirst() - .getColumns() - .keySet()) { - headers.addDoubleColumn(s); - } - - for (int ii = 0; ii < 5; ii++) { - headers.addStringColumn("remark_" + ii); - } - - return headers.headers; - } - - private static void addEmissionsAnnualOverallBenefit(StreetBaseDataContainer baseDataContainer, - List record, Emission emission) { - record.add(baseDataContainer.getPhysicalEffect() - .getKfzEmission(emission)); - record.add(Optional.ofNullable(baseDataContainer.getCostBenefitAnalysis()) - .map(StreetCostBenefitAnalysisDataContainer::getNa) - .map(m -> m.get(emission)) - .map(Benefit::annual) - .orElse(null)); - record.add(Optional.ofNullable(baseDataContainer.getCostBenefitAnalysis()) - .map(StreetCostBenefitAnalysisDataContainer::getNa) - .map(m -> m.get(emission)) - .map(Benefit::overall) - .orElse(null)); - } -} diff --git a/src/main/java/org/tub/vsp/bvwp/io/RailTableCreator.java b/src/main/java/org/tub/vsp/bvwp/io/RailTableCreator.java new file mode 100644 index 0000000..6272d01 --- /dev/null +++ b/src/main/java/org/tub/vsp/bvwp/io/RailTableCreator.java @@ -0,0 +1,175 @@ +package org.tub.vsp.bvwp.io; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.tub.vsp.bvwp.data.Headers; +import org.tub.vsp.bvwp.data.container.analysis.RailAnalysisDataContainer; +import org.tub.vsp.bvwp.data.container.base.rail.RailBaseDataContainer; +import tech.tablesaw.api.DoubleColumn; +import tech.tablesaw.api.Row; +import tech.tablesaw.api.StringColumn; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class RailTableCreator { + private static final Logger logger = LogManager.getLogger(StreetCsvWriter.class); + + private final Table table; + + public RailTableCreator() { + this.table = Table.create("defaultTable"); + // (I am somewhat abusing this class to also generate a TableSaw Table. Probably would make sense later to + // first genrate the Table object, + // and then csv-write that. kai, feb'24) + } + + public Table computeTable(List analysisDataContainers) { + addHeaders(analysisDataContainers); + + for (RailAnalysisDataContainer analysisDataContainer : analysisDataContainers) { + getCsvRecord(analysisDataContainer, table); + } + return table; + } + + private static class Record { + private final Row row; + private final List record = new ArrayList<>(); + private final Table table; + int ii = 0; + + private Record(Table table) { + this.row = table.appendRow(); + this.table = table; + } + + // private Record add( String str ) { +// row.setString( ii, str ); +// record.add( str ); +// ii++; +// return this; +// } + private Record add(String key, String str) { + +// if ( !table.columnNames().contains( key ) ) { +// table.addColumns( StringColumn.create( key ) ); +// } + // not working (I think) + + row.setString(key, str); + record.add(str); + ii++; + return this; + } + + // private Record add( Double dbl ) { +// if ( dbl != null ){ +// row.setDouble( ii, dbl ); +// } else { +// row.setDouble( ii, Double.NaN ); +// } +// record.add( dbl ); +// ii++; +// return this; +// } + private Record add(String key, Double dbl) { + +// if ( !table.columnNames().contains( key ) ) { +// table.addColumns( DoubleColumn.create( key ) ); +// } + // not working (I think) + + if (dbl != null) { + row.setDouble(key, dbl); + } else { + row.setDouble(key, Double.NaN); + } + record.add(dbl); + ii++; + return this; + } + } + + private static List getCsvRecord( RailAnalysisDataContainer analysisDataContainer, Table table ) { + RailBaseDataContainer baseDataContainer = analysisDataContainer.getBaseDataContainer(); + + Record record = new Record(table); + + //general info + record.add(Headers.PROJECT_NAME, baseDataContainer.getProjectInformation().getProjectNumber()); + record.add(Headers.LINK, baseDataContainer.getUrl()); + record.add(Headers.EINSTUFUNG, baseDataContainer.getProjectInformation().getPriority().name() ); + + record.add(Headers.LENGTH, baseDataContainer.getProjectInformation().getLength()); + + //overall benefit and cost + record.add(Headers.B_OVERALL, baseDataContainer.getCostBenefitAnalysis().getOverallBenefit().overall()); + record.add(Headers.INVCOST_ORIG, baseDataContainer.getCostBenefitAnalysis().getCost().overallCosts()); + + // (yy warum diese aufwändige Syntax? kai, feb'24) + // --> da sowohl getCostBenefitAnalysis, getCost als auch overallCosts null zurückgeben können, wenn die + // Daten nicht vorhanden sind. So spart man sich null checks (paul, feb'24) + + for (Map.Entry entry : analysisDataContainer.getColumns().entrySet()) { + record.add(entry.getKey(), entry.getValue()); + } + + int ii = 0; + for (String remark : analysisDataContainer.getRemarks()) { + record.add("remark_" + ii, "\"" + remark + "\""); + ii++; + } + + return record.record; + } + + private void addHeaders(List analysisDataContainers) { + //assert that all entries of new nkv have the same keys + assert analysisDataContainers.stream() + .allMatch(a -> { + Set thisKeys = a.getColumns().keySet(); + Set firstKeys = analysisDataContainers.getFirst().getColumns() + .keySet(); + return thisKeys.containsAll(firstKeys) && thisKeys.size() == firstKeys.size(); + }) : "Not all nkv have the same keys"; + + table.addColumns(StringColumn.create(Headers.PROJECT_NAME)); + + table.addColumns(StringColumn.create(Headers.LINK)); + table.addColumns(StringColumn.create(Headers.EINSTUFUNG)); + table.addColumns(StringColumn.create(Headers.BAUTYP)); + table.addColumns(DoubleColumn.create(Headers.LENGTH)); + + table.addColumns(DoubleColumn.create(Headers.ADDTL_PKWKM_ORIG)); + table.addColumns(DoubleColumn.create(Headers.ADDTL_PKWKM_INDUZ_ORIG)); + table.addColumns(DoubleColumn.create(Headers.B_FZKM)); + + table.addColumns(DoubleColumn.create(Headers.CO_2_EQUIVALENTS_EMISSIONS)); + // headers.addDoubleColumn( Headers.B_CO_2_EQUIVALENTS_ANNUAL ); + table.addColumns(DoubleColumn.create(Headers.B_CO_2_EQUIVALENTS_ORIG)); + +// for (String colName : EMISSION_COLUMNS.values()) { +// headers.add(colName + "-emissions"); +// headers.add(colName + "-annual"); +// headers.add(colName + "-overall"); +// } + + table.addColumns(DoubleColumn.create(Headers.B_OVERALL)); + table.addColumns(DoubleColumn.create(Headers.INVCOST_ORIG)); + + for (String s : analysisDataContainers.getFirst() + .getColumns() + .keySet()) { + table.addColumns(DoubleColumn.create(s)); + } + + for (int ii = 0; ii < 5; ii++) { + table.addColumns(StringColumn.create("remark_" + ii)); + } + + } +} diff --git a/src/main/java/org/tub/vsp/bvwp/scraping/RailScraper.java b/src/main/java/org/tub/vsp/bvwp/scraping/RailScraper.java index 6ac8b59..f1fe076 100644 --- a/src/main/java/org/tub/vsp/bvwp/scraping/RailScraper.java +++ b/src/main/java/org/tub/vsp/bvwp/scraping/RailScraper.java @@ -3,6 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; +import org.tub.vsp.bvwp.JSoupUtils; import org.tub.vsp.bvwp.data.container.base.rail.RailBaseDataContainer; import org.tub.vsp.bvwp.data.mapper.costBenefit.RailCostBenefitMapper; import org.tub.vsp.bvwp.data.mapper.physicalEffect.RailPhysicalEffectMapper; @@ -50,7 +51,8 @@ public List extractAllLocalBaseData(String projectFolder, } private boolean isProjectScrapable(Document doc) { - //if there is only one table, there is no further information to scrape (e.g. https://www.bvwp-projekte.de/schiene/2-034-V01/2-034-V01.html) - return doc.select("table.table_grunddaten").size() > 1; + //if there is no emissions table, there is no further information to scrape (e.g. https://www.bvwp-projekte.de/schiene/2-034-V01/2-034-V01.html) + return JSoupUtils.getTableByKeyAndContainedText(doc, "table.table_webprins", + "Veränderung der Abgasemissionen").isPresent(); } } diff --git a/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java b/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java index ae31b5a..69b70a4 100644 --- a/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java +++ b/src/main/java/org/tub/vsp/bvwp/users/kn/RunLocalRailScrapingKN.java @@ -3,7 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.tub.vsp.bvwp.data.container.analysis.RailAnalysisDataContainer; -import org.tub.vsp.bvwp.io.RailCsvWriter; +import org.tub.vsp.bvwp.io.RailTableCreator; import org.tub.vsp.bvwp.scraping.RailScraper; import tech.tablesaw.api.Table; @@ -22,9 +22,8 @@ public static void main(String[] args) { .toList(); logger.info("Writing csv"); - RailCsvWriter csvWriter = new RailCsvWriter("output/rail_data.csv"); - Table table = csvWriter.writeCsv( allRailData ); + RailTableCreator tableCreator = new RailTableCreator(); - //TODO + Table table = tableCreator.computeTable(allRailData); } } From 100df5e1787027dbe2ae3ac50736c4ae293159a9 Mon Sep 17 00:00:00 2001 From: Paul Heinrich Date: Fri, 31 May 2024 13:59:48 +0200 Subject: [PATCH 5/5] add test --- .../bvwp/computation/NkvCalculatorRail.java | 9 ++++---- .../computation/NkvCalculatorRailTest.java | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 src/test/java/org/tub/vsp/bvwp/computation/NkvCalculatorRailTest.java diff --git a/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculatorRail.java b/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculatorRail.java index 8e32096..b3d6632 100644 --- a/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculatorRail.java +++ b/src/main/java/org/tub/vsp/bvwp/computation/NkvCalculatorRail.java @@ -28,12 +28,13 @@ public static Double calculateNkv(Modifications modifications, RailBaseDataConta assert modifications.mehrFzkm() == 0; assert modifications.nonCo2BenefitsFactor() == 1.; - double baukosten = railBaseDataContainer.getCostBenefitAnalysis().getCost().overallCosts() * modifications.constructionCostFactor(); - double benefit = railBaseDataContainer.getCostBenefitAnalysis().getOverallBenefit().overall(); - double co2_infra_eur = railBaseDataContainer.getCostBenefitAnalysis().getNl().overall(); + double baukosten_MioEur = railBaseDataContainer.getCostBenefitAnalysis().getCost().overallCosts() * modifications.constructionCostFactor(); + double benefit_MioEur = railBaseDataContainer.getCostBenefitAnalysis().getOverallBenefit().overall(); + double co2_infra_MioEur = railBaseDataContainer.getCostBenefitAnalysis().getNl().overall(); double co2_betrieb_t = railBaseDataContainer.getPhysicalEffect().getEmissionsDataContainer().emissions().get(Emission.CO2); - return ComputationKN.nkv_rail(modifications.co2Price(), baukosten, benefit, co2_infra_eur, co2_betrieb_t); + return ComputationKN.nkv_rail(modifications.co2Price(), baukosten_MioEur * 10e6, benefit_MioEur * 10e6, co2_infra_MioEur * 10e6, + co2_betrieb_t); } // @Deprecated // use instance approach diff --git a/src/test/java/org/tub/vsp/bvwp/computation/NkvCalculatorRailTest.java b/src/test/java/org/tub/vsp/bvwp/computation/NkvCalculatorRailTest.java new file mode 100644 index 0000000..2e51c68 --- /dev/null +++ b/src/test/java/org/tub/vsp/bvwp/computation/NkvCalculatorRailTest.java @@ -0,0 +1,22 @@ +package org.tub.vsp.bvwp.computation; + +import org.junit.jupiter.api.Test; +import org.tub.vsp.bvwp.data.LocalFileAccessor; +import org.tub.vsp.bvwp.data.container.base.rail.RailBaseDataContainer; +import org.tub.vsp.bvwp.scraping.RailScraper; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class NkvCalculatorRailTest { + @Test + void test_rail() throws IOException { + RailScraper railScraper = new RailScraper(); + RailBaseDataContainer railBaseDataContainer = railScraper.extractBaseData(LocalFileAccessor.getLocalDocument("2-003-v01.html")).orElseThrow(); + + var nkvCalculator = new NkvCalculatorRail(railBaseDataContainer); + Double calculateNkv = nkvCalculator.calculateNkv(Modifications.NO_CHANGE); + assertEquals(2.1593, calculateNkv, 10e-4); + } +} \ No newline at end of file