Merge pull request #24 from matsim-vsp/complete-rail-scraping

Complete rail scraping
matsim-vsp · May 24, 2024 · cae9594 · cae9594
2 parents 34d875f + 63bb16c
commit cae9594
Show file tree

Hide file tree

Showing 21 changed files with 716 additions and 134 deletions.
diff --git a/src/main/java/org/tub/vsp/bvwp/JSoupUtils.java b/src/main/java/org/tub/vsp/bvwp/JSoupUtils.java
@@ -58,6 +58,13 @@ public static Optional<Element> firstRowWithKeyContainedInCol(Element table, Str
                     .findFirst();
     }
 
+    public static Optional<Element> firstRowWithKeyMatchesInCol(Element table, String key, int colIndex) {
+        return table.select("tr")
+                    .stream()
+                    .filter(r -> r.child(colIndex).text().equals(key))
+                    .findFirst();
+    }
+
     public static Optional<Element> firstRowWithKeyInCol(Element table, String key, int colIndex) {
         return table.select("tr")
                     .stream()

diff --git a/src/main/java/org/tub/vsp/bvwp/RunLocalRailScraping.java b/src/main/java/org/tub/vsp/bvwp/RunLocalRailScraping.java
@@ -0,0 +1,28 @@
+package org.tub.vsp.bvwp;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.tub.vsp.bvwp.data.container.analysis.RailAnalysisDataContainer;
+import org.tub.vsp.bvwp.io.RailCsvWriter;
+import org.tub.vsp.bvwp.scraping.RailScraper;
+
+import java.util.List;
+
+public class RunLocalRailScraping {
+    private static final Logger logger = LogManager.getLogger(RunLocalRailScraping.class);
+
+    public static void main(String[] args) {
+        RailScraper scraper = new RailScraper();
+
+        logger.info("Starting scraping");
+        List<RailAnalysisDataContainer> allRailData = scraper.extractAllLocalBaseData("./data/rail/all", "", "^2.*", "")
+                                                             .stream()
+                                                             .map(RailAnalysisDataContainer::new)
+                                                             .toList();
+
+        logger.info("Writing csv");
+        RailCsvWriter csvWriter = new RailCsvWriter();
+
+        //TODO
+    }
+}
diff --git a/...org/tub/vsp/bvwp/RunLocalCsvScraping.java → ...b/vsp/bvwp/RunLocalStreetCsvScraping.java b/...org/tub/vsp/bvwp/RunLocalCsvScraping.java → ...b/vsp/bvwp/RunLocalStreetCsvScraping.java
@@ -8,8 +8,8 @@
 
 import java.util.List;
 
-public class RunLocalCsvScraping {
-    private static final Logger logger = LogManager.getLogger(RunLocalCsvScraping.class);
+public class RunLocalStreetCsvScraping {
+    private static final Logger logger = LogManager.getLogger(RunLocalStreetCsvScraping.class);
 
     public static void main(String[] args) {
         logger.warn("(vermutl. weitgehend gelöst) Teilweise werden die Hauptprojekte bewertet und nicht die " +

diff --git a/src/main/java/org/tub/vsp/bvwp/data/container/analysis/RailAnalysisDataContainer.java b/src/main/java/org/tub/vsp/bvwp/data/container/analysis/RailAnalysisDataContainer.java
@@ -0,0 +1,17 @@
+package org.tub.vsp.bvwp.data.container.analysis;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.tub.vsp.bvwp.data.container.base.rail.RailBaseDataContainer;
+
+public class RailAnalysisDataContainer {
+    Logger logger = LogManager.getLogger(RailAnalysisDataContainer.class);
+
+    private final RailBaseDataContainer baseDataContainer;
+
+    public RailAnalysisDataContainer(RailBaseDataContainer baseDataContainer) {
+        this.baseDataContainer = baseDataContainer;
+    }
+
+    //add analysis stuff here...
+}
diff --git a/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailBaseDataContainer.java b/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailBaseDataContainer.java
@@ -1,5 +1,79 @@
 package org.tub.vsp.bvwp.data.container.base.rail;
 
+import java.util.Objects;
+
 public class RailBaseDataContainer {
+    private String url;
+
+    RailProjectInformationDataContainer projectInformation;
+    RailPhysicalEffectDataContainer physicalEffect;
+    RailCostBenefitAnalysisDataContainer costBenefitAnalysis;
+
+    public String getUrl() {
+        return url;
+    }
+
+    public RailBaseDataContainer setUrl(String url) {
+        this.url = url;
+        return this;
+    }
+
+    public RailProjectInformationDataContainer getProjectInformation() {
+        return projectInformation;
+    }
+
+    public RailBaseDataContainer setProjectInformation(RailProjectInformationDataContainer projectInformation) {
+        this.projectInformation = projectInformation;
+        return this;
+    }
+
+    public RailPhysicalEffectDataContainer getPhysicalEffect() {
+        return physicalEffect;
+    }
+
+    public RailBaseDataContainer setPhysicalEffect(RailPhysicalEffectDataContainer physicalEffect) {
+        this.physicalEffect = physicalEffect;
+        return this;
+    }
+
+    public RailCostBenefitAnalysisDataContainer getCostBenefitAnalysis() {
+        return costBenefitAnalysis;
+    }
+
+    public RailBaseDataContainer setCostBenefitAnalysis(RailCostBenefitAnalysisDataContainer costBenefitAnalysis) {
+        this.costBenefitAnalysis = costBenefitAnalysis;
+        return this;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        RailBaseDataContainer that = (RailBaseDataContainer) o;
+
+        if (!Objects.equals(url, that.url)) {
+            return false;
+        }
+        if (!Objects.equals(projectInformation, that.projectInformation)) {
+            return false;
+        }
+        if (!Objects.equals(physicalEffect, that.physicalEffect)) {
+            return false;
+        }
+        return Objects.equals(costBenefitAnalysis, that.costBenefitAnalysis);
+    }
 
+    @Override
+    public int hashCode() {
+        int result = url != null ? url.hashCode() : 0;
+        result = 31 * result + (projectInformation != null ? projectInformation.hashCode() : 0);
+        result = 31 * result + (physicalEffect != null ? physicalEffect.hashCode() : 0);
+        result = 31 * result + (costBenefitAnalysis != null ? costBenefitAnalysis.hashCode() : 0);
+        return result;
+    }
 }
diff --git a/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailBenefitFreightDataContainer.java b/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailBenefitFreightDataContainer.java
@@ -2,6 +2,8 @@
 
 import org.tub.vsp.bvwp.data.type.Benefit;
 
+import java.util.Objects;
+
 public class RailBenefitFreightDataContainer {
     private Benefit nbLkw;
     private Benefit nbSchiene;
@@ -169,4 +171,84 @@ public RailBenefitFreightDataContainer setOverallBenefit(Benefit overallBenefit)
         this.overallBenefit = overallBenefit;
         return this;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        RailBenefitFreightDataContainer that = (RailBenefitFreightDataContainer) o;
+
+        if (!Objects.equals(nbLkw, that.nbLkw)) {
+            return false;
+        }
+        if (!Objects.equals(nbSchiene, that.nbSchiene)) {
+            return false;
+        }
+        if (!Objects.equals(nbSchiff, that.nbSchiff)) {
+            return false;
+        }
+        if (!Objects.equals(naLkw, that.naLkw)) {
+            return false;
+        }
+        if (!Objects.equals(naSchiene, that.naSchiene)) {
+            return false;
+        }
+        if (!Objects.equals(naSchiff, that.naSchiff)) {
+            return false;
+        }
+        if (!Objects.equals(nsLkw, that.nsLkw)) {
+            return false;
+        }
+        if (!Objects.equals(nsSchiene, that.nsSchiene)) {
+            return false;
+        }
+        if (!Objects.equals(nsSchiff, that.nsSchiff)) {
+            return false;
+        }
+        if (!Objects.equals(ntzVerbVerkehr, that.ntzVerbVerkehr)) {
+            return false;
+        }
+        if (!Objects.equals(ntzLkwSchiene, that.ntzLkwSchiene)) {
+            return false;
+        }
+        if (!Objects.equals(ntzSchiffSchiene, that.ntzSchiffSchiene)) {
+            return false;
+        }
+        if (!Objects.equals(niLkwSchiene, that.niLkwSchiene)) {
+            return false;
+        }
+        if (!Objects.equals(niSchiffSchiene, that.niSchiffSchiene)) {
+            return false;
+        }
+        if (!Objects.equals(nzVerbVerkehr, that.nzVerbVerkehr)) {
+            return false;
+        }
+        return Objects.equals(overallBenefit, that.overallBenefit);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = nbLkw != null ? nbLkw.hashCode() : 0;
+        result = 31 * result + (nbSchiene != null ? nbSchiene.hashCode() : 0);
+        result = 31 * result + (nbSchiff != null ? nbSchiff.hashCode() : 0);
+        result = 31 * result + (naLkw != null ? naLkw.hashCode() : 0);
+        result = 31 * result + (naSchiene != null ? naSchiene.hashCode() : 0);
+        result = 31 * result + (naSchiff != null ? naSchiff.hashCode() : 0);
+        result = 31 * result + (nsLkw != null ? nsLkw.hashCode() : 0);
+        result = 31 * result + (nsSchiene != null ? nsSchiene.hashCode() : 0);
+        result = 31 * result + (nsSchiff != null ? nsSchiff.hashCode() : 0);
+        result = 31 * result + (ntzVerbVerkehr != null ? ntzVerbVerkehr.hashCode() : 0);
+        result = 31 * result + (ntzLkwSchiene != null ? ntzLkwSchiene.hashCode() : 0);
+        result = 31 * result + (ntzSchiffSchiene != null ? ntzSchiffSchiene.hashCode() : 0);
+        result = 31 * result + (niLkwSchiene != null ? niLkwSchiene.hashCode() : 0);
+        result = 31 * result + (niSchiffSchiene != null ? niSchiffSchiene.hashCode() : 0);
+        result = 31 * result + (nzVerbVerkehr != null ? nzVerbVerkehr.hashCode() : 0);
+        result = 31 * result + (overallBenefit != null ? overallBenefit.hashCode() : 0);
+        return result;
+    }
 }
diff --git a/.../java/org/tub/vsp/bvwp/data/container/base/rail/RailCostBenefitAnalysisDataContainer.java b/.../java/org/tub/vsp/bvwp/data/container/base/rail/RailCostBenefitAnalysisDataContainer.java
@@ -5,12 +5,16 @@
 import org.tub.vsp.bvwp.data.type.Benefit;
 import org.tub.vsp.bvwp.data.type.Cost;
 
+import java.util.Objects;
+
 public class RailCostBenefitAnalysisDataContainer {
     private static final Logger logger = LogManager.getLogger(RailCostBenefitAnalysisDataContainer.class);
 
     private RailBenefitPassengerDataContainer passengerBenefits;
     private RailBenefitFreightDataContainer freightBenefits;
 
+    private Benefit nl;
+
     private Benefit overallBenefit;
     private Cost cost;
 
@@ -49,4 +53,49 @@ public RailCostBenefitAnalysisDataContainer setCost(Cost cost) {
         this.cost = cost;
         return this;
     }
+
+    public Benefit getNl() {
+        return nl;
+    }
+
+    public RailCostBenefitAnalysisDataContainer setNl(Benefit nl) {
+        this.nl = nl;
+        return this;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        RailCostBenefitAnalysisDataContainer that = (RailCostBenefitAnalysisDataContainer) o;
+
+        if (!Objects.equals(passengerBenefits, that.passengerBenefits)) {
+            return false;
+        }
+        if (!Objects.equals(freightBenefits, that.freightBenefits)) {
+            return false;
+        }
+        if (!Objects.equals(nl, that.nl)) {
+            return false;
+        }
+        if (!Objects.equals(overallBenefit, that.overallBenefit)) {
+            return false;
+        }
+        return Objects.equals(cost, that.cost);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = passengerBenefits != null ? passengerBenefits.hashCode() : 0;
+        result = 31 * result + (freightBenefits != null ? freightBenefits.hashCode() : 0);
+        result = 31 * result + (nl != null ? nl.hashCode() : 0);
+        result = 31 * result + (overallBenefit != null ? overallBenefit.hashCode() : 0);
+        result = 31 * result + (cost != null ? cost.hashCode() : 0);
+        return result;
+    }
 }
diff --git a/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailPhysicalEffectDataContainer.java b/src/main/java/org/tub/vsp/bvwp/data/container/base/rail/RailPhysicalEffectDataContainer.java
@@ -1,5 +1,7 @@
 package org.tub.vsp.bvwp.data.container.base.rail;
 
+import java.util.Objects;
+
 public class RailPhysicalEffectDataContainer {
     private RailEmissionsDataContainer emissionsDataContainer;
 
@@ -11,4 +13,23 @@ public RailPhysicalEffectDataContainer setEmissionsDataContainer(RailEmissionsDa
         this.emissionsDataContainer = emissionsDataContainer;
         return this;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        RailPhysicalEffectDataContainer that = (RailPhysicalEffectDataContainer) o;
+
+        return Objects.equals(emissionsDataContainer, that.emissionsDataContainer);
+    }
+
+    @Override
+    public int hashCode() {
+        return emissionsDataContainer != null ? emissionsDataContainer.hashCode() : 0;
+    }
 }