diff --git a/.gitignore b/.gitignore index e58be71..6bd25c3 100644 --- a/.gitignore +++ b/.gitignore @@ -23,26 +23,18 @@ # environment. # Manifest.toml -# my things vari out* .DS_Store -# data/*.csv -# data/*.gz -# data/OeBB -# data/data_password p?pp* *.ini -# hidden_data/ -# scripts/help.txt +*.pdf +*.csv +*.dat +*.agr +*.png +*.zip data/ old/ .vscode/ -# Simone things -# data/img -# data/delays -# postprocessing # commented by Vito -preprocessing/trainIni.in -preprocessing/train_starts.ipynb -map/ diff --git a/configuration/configure.jl b/configuration/configure.jl index 6468001..b93d239 100644 --- a/configuration/configure.jl +++ b/configuration/configure.jl @@ -3,16 +3,8 @@ configure.jl OLD README, must be changed Input: - 1) a csv file with the daily PAD Zuglaufdaten as provided by OeBB - 2) a csv file with the preprocessed xml containing the yearly scheduled timetable - 3) a csv file with the blocks found from the RINF data: "rinf-blocks.csv" - 4) a csv file with the operational points determined from RINF: "rinf-OperationalPoints.csv" - outfile = "blocks.csv" + Output: - 1) a csv file with the timetable to use in the simulation: "timetable.csv" - 2) a csv file with the list of blocks for the simulation: "blocks.csv" - 3) a csv file with the list of operational points with more than one track (stations, junctions) - - an intermediate file "blocks-xml-YEAR.csv" distilled from RINF and XML Description: The task of this script is to cure the many issues present in data. @@ -91,7 +83,7 @@ source_path::String = parsed_args["source_data_path"] target_path::String = parsed_args["target_data_path"] nr_exo_delays::Int = parsed_args["exo_delays"]; delays_only::Bool = parsed_args["delays_only"]; -# use_real_time = parsed_args["use_real_time"]; +use_real_time = parsed_args["use_real_time"]; find_rotations::Bool = parsed_args["rotations"]; pad_schedule::Bool = parsed_args["pad_schedule"]; select_line::String = parsed_args["select_line"]; @@ -173,6 +165,9 @@ function loadPAD(file::String)::DataFrame types = String, skipto = 2) |> DataFrame; + # use a dirty trick to work with real time instead of scheduled + use_real_time && rename!(bigpad, :scheduledtime => :unusedtime, :realtime => :scheduledtime); + dropmissing!(bigpad, :scheduledtime); filter!(x->length(x.scheduledtime)>0, bigpad ); diff --git a/configuration/exo_delays.jl b/configuration/exo_delays.jl index b9290c6..c33c65a 100644 --- a/configuration/exo_delays.jl +++ b/configuration/exo_delays.jl @@ -48,8 +48,9 @@ function SampleExoDelays( @info "\tLoading data." dfn=DataFrame(CSV.File(fileNdelay, select=[:number])); - dftbl=DataFrame(CSV.File(fileTimetable, select=[:trainid])); # Or Mon, Tue, ... Or timetable + dftbl=DataFrame(CSV.File(fileTimetable, select=[:trainid, :bst])); # Or Mon, Tue, ... Or timetable trains=unique(dftbl.trainid); + ops = unique(dftbl.bst); df=DataFrame(CSV.File(fileDelayList)); @@ -72,7 +73,7 @@ function SampleExoDelays( sample_row_idxs = rand(1:dfnrow, effective_n); # It samples n rows dfout = df[sample_row_idxs, :]; unique!(dfout); - filter!(x-> x.trainid ∈ trains, dfout); + filter!(x-> (x.trainid ∈ trains)&&(x.block ∈ ops), dfout); # REMINDER: check when there is a blk instead of a station @info "\tFile nr $i, $n delays required, $(nrow(dfout)) delays created instead ($(round(Int,100*nrow(dfout)/n))%)"; sequence = lpad(i,4,"0"); outfile = "$(baseoutfile)_$sequence" * "$extension"; diff --git a/configuration/parser.jl b/configuration/parser.jl index f2b9438..8320b5b 100644 --- a/configuration/parser.jl +++ b/configuration/parser.jl @@ -45,10 +45,9 @@ function parse_commandline() help = "Skip *ALL* the configuration actions except the sampling of delays." action = :store_true - - # "--use_real_time" - # help = "Use the real time column of the PAD timetable instead of the scheduled time." - # action = :store_true + "--use_real_time" + help = "Use the real time column of the PAD timetable instead of the scheduled time." + action = :store_true "--pad_schedule" help = "Use the scheduled time in the PAD file instead of the scheduled time found in XML." diff --git a/environment/Manifest.toml b/environment/Manifest.toml index 56221b2..455c36e 100644 --- a/environment/Manifest.toml +++ b/environment/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.9.2" +julia_version = "1.9.3" manifest_format = "2.0" project_hash = "af233d7f8b553209c7409fda7c02d46300d5c3ee" @@ -33,15 +33,15 @@ version = "0.10.11" [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "9c209fb7536406834aa938fb149964b985de6c83" +git-tree-sha1 = "cd67fc487743b2f0fd4380d4cbd3a24660d0eec8" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.1" +version = "0.7.3" [[deps.Compat]] deps = ["UUIDs"] -git-tree-sha1 = "4e88377ae7ebeaf29a047aa1ee40826e0b708a5d" +git-tree-sha1 = "8a62af3e248a8c4bad6b32cbbe663ae02275e32c" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.7.0" +version = "4.10.0" weakdeps = ["Dates", "LinearAlgebra"] [deps.Compat.extensions] @@ -54,9 +54,9 @@ version = "1.0.5+0" [[deps.ConcurrentUtilities]] deps = ["Serialization", "Sockets"] -git-tree-sha1 = "96d823b94ba8d187a6d8f0826e731195a74b90e9" +git-tree-sha1 = "5372dbbf8f0bdb8c700db5367132925c0771ef7e" uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" -version = "2.2.0" +version = "2.2.1" [[deps.Coverage]] deps = ["CoverageTools", "HTTP", "JSON", "LibGit2", "MbedTLS"] @@ -80,16 +80,16 @@ uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" version = "1.15.0" [[deps.DataFrames]] -deps = ["Compat", "DataAPI", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrettyTables", "Printf", "REPL", "Random", "Reexport", "SentinelArrays", "SnoopPrecompile", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] -git-tree-sha1 = "aa51303df86f8626a962fccb878430cdb0a97eee" +deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "REPL", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "04c738083f29f86e62c8afc341f0967d8717bdb8" uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -version = "1.5.0" +version = "1.6.1" [[deps.DataStructures]] deps = ["Compat", "InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "cf25ccb972fec4e4817764d01c82386ae94f77b4" +git-tree-sha1 = "3dbd312d370723b6bb43ba9d02fc36abade4518d" uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.18.14" +version = "0.18.15" [[deps.DataValueInterfaces]] git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" @@ -113,28 +113,22 @@ version = "0.1.9" [[deps.FilePathsBase]] deps = ["Compat", "Dates", "Mmap", "Printf", "Test", "UUIDs"] -git-tree-sha1 = "e27c4ebe80e8699540f2d6c805cc12203b614f12" +git-tree-sha1 = "9f00e42f8d99fdde64d40c8ea5d14269a2e2c1aa" uuid = "48062228-2e41-5def-b9a4-89aafe57970f" -version = "0.9.20" +version = "0.9.21" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" -[[deps.Formatting]] -deps = ["Printf"] -git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8" -uuid = "59287772-0a20-5a39-b81b-1366585eb4c0" -version = "0.4.2" - [[deps.Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" [[deps.HTTP]] deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] -git-tree-sha1 = "7f5ef966a02a8fdf3df2ca03108a88447cb3c6f0" +git-tree-sha1 = "5eab648309e2e060198b45820af1a37182de3cce" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "1.9.8" +version = "1.10.0" [[deps.InlineStrings]] deps = ["Parsers"] @@ -157,10 +151,10 @@ uuid = "82899510-4779-5014-852e-03e436cf321d" version = "1.0.0" [[deps.JLLWrappers]] -deps = ["Preferences"] -git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.4.1" +version = "1.5.0" [[deps.JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] @@ -204,9 +198,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[deps.LoggingExtras]] deps = ["Dates", "Logging"] -git-tree-sha1 = "cedb76b37bc5a6c702ade66be44f831fa23c681e" +git-tree-sha1 = "c1dd6d7978c12545b4179fb6153b9250c96b0075" uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" -version = "1.0.0" +version = "1.0.3" [[deps.Markdown]] deps = ["Base64"] @@ -253,20 +247,20 @@ version = "1.4.1" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "cae3153c7f6cf3f069a853883fd1919a6e5bab5b" +git-tree-sha1 = "ceeda72c9fd6bbebc4f4f598560789145a8b6c4c" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.9+0" +version = "3.0.11+0" [[deps.OrderedCollections]] -git-tree-sha1 = "d321bf2de576bf25ec4d3e4360faca399afca282" +git-tree-sha1 = "2e73fe17cac3c62ad1aebe70d44c963c3cfdc3e3" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.6.0" +version = "1.6.2" [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "4b2e829ee66d4218e0cef22c0a64ee37cf258c29" +git-tree-sha1 = "716e24b21538abc91f6205fd1d8363f39b442851" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.7.1" +version = "2.7.2" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] @@ -275,21 +269,21 @@ version = "1.9.2" [[deps.PooledArrays]] deps = ["DataAPI", "Future"] -git-tree-sha1 = "a6062fe4063cdafe78f4a0a81cfffb89721b30e7" +git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" -version = "1.4.2" +version = "1.4.3" [[deps.PrecompileTools]] deps = ["Preferences"] -git-tree-sha1 = "9673d39decc5feece56ef3940e5dafba15ba0f81" +git-tree-sha1 = "03b4c25b43cb84cee5c90aa9b5ea0a78fd848d2f" uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.1.2" +version = "1.2.0" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "7eb1686b4f04b82f96ed7a4ea5890a4f0c7a09f1" +git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.0" +version = "1.4.1" [[deps.PrettyPrint]] git-tree-sha1 = "632eb4abab3449ab30c5e1afaa874f0b98b586e4" @@ -297,10 +291,10 @@ uuid = "8162dcfd-2161-5ef2-ae6c-7681170c5f98" version = "0.2.0" [[deps.PrettyTables]] -deps = ["Crayons", "Formatting", "LaTeXStrings", "Markdown", "Reexport", "StringManipulation", "Tables"] -git-tree-sha1 = "213579618ec1f42dea7dd637a42785a608b1ea9c" +deps = ["Crayons", "LaTeXStrings", "Markdown", "Printf", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "6842ce83a836fbbc0cfeca0b5a4de1a4dcbdb8d1" uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "2.2.4" +version = "2.2.8" [[deps.Printf]] deps = ["Unicode"] @@ -337,20 +331,14 @@ git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1" uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" version = "1.1.0" -[[deps.SnoopPrecompile]] -deps = ["Preferences"] -git-tree-sha1 = "e760a70afdcd461cf01a575947738d359234665c" -uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c" -version = "1.0.3" - [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" [[deps.SortingAlgorithms]] deps = ["DataStructures"] -git-tree-sha1 = "c60ec5c62180f27efea3ba2908480f8055e17cee" +git-tree-sha1 = "5165dfb9fd131cf0c6957a3a7605dede376e7b63" uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.1.1" +version = "1.2.0" [[deps.SparseArrays]] deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] @@ -362,9 +350,10 @@ uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" version = "1.9.0" [[deps.StringManipulation]] -git-tree-sha1 = "46da2434b41f41ac3594ee9816ce5541c6096123" +deps = ["PrecompileTools"] +git-tree-sha1 = "a04cabe79c5f01f4d723cc6704070ada0b9d46d5" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" -version = "0.3.0" +version = "0.3.4" [[deps.SuiteSparse_jll]] deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] @@ -383,10 +372,10 @@ uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" version = "1.0.1" [[deps.Tables]] -deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"] -git-tree-sha1 = "1544b926975372da01227b382066ab70e574a3ec" +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits"] +git-tree-sha1 = "cb76cf677714c095e535e3501ac7954732aeea2d" uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "1.10.1" +version = "1.11.1" [[deps.Tar]] deps = ["ArgTools", "SHA"] @@ -403,15 +392,18 @@ uuid = "b718987f-49a8-5099-9789-dcd902bef87d" version = "1.0.1" [[deps.TranscodingStreams]] -deps = ["Random", "Test"] -git-tree-sha1 = "9a6ae7ed916312b41236fcef7e0af564ef934769" +git-tree-sha1 = "49cbf7c74fafaed4c529d47d48c8f7da6a19eb75" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.9.13" +version = "0.10.1" +weakdeps = ["Random", "Test"] + + [deps.TranscodingStreams.extensions] + TestExt = ["Test", "Random"] [[deps.URIs]] -git-tree-sha1 = "074f993b0ca030848b897beff716d93aca60f06a" +git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b" uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -version = "1.4.2" +version = "1.5.1" [[deps.UUIDs]] deps = ["Random", "SHA"] diff --git a/postprocessing/csv2dat.jl b/postprocessing/csv2dat.jl new file mode 100644 index 0000000..ed2ff89 --- /dev/null +++ b/postprocessing/csv2dat.jl @@ -0,0 +1,12 @@ +using DataFrames,CSV; + +filelist = readdir("."); +filter!(endswith(".csv"), filelist); + +for file in filelist + # file = filelist[1] + df = CSV.read(file, DataFrame); + outfile = splitext(file)[1]*".dat"; + CSV.write(outfile, df, header=false, delim=" "); +end + diff --git a/postprocessing/delays_at_final_dest.jl b/postprocessing/delays_at_final_dest.jl new file mode 100644 index 0000000..eae846b --- /dev/null +++ b/postprocessing/delays_at_final_dest.jl @@ -0,0 +1,93 @@ +""" +Takes two folder containing the csv files of simulation output +and determines and plots the delays at destination. +""" + + + +using CSV, DataFrames, Plots; + +function cumul!(V::Vector{Int}) + filter!(x->x>0, V); + sort!(V) + r = collect(length(V):-1:1); + r = r./r[1]; + (V, r) +end + +function main(dir1="/Users/servedio/prgs/training/simulation/data/rotations", + dir2= "/Users/servedio/prgs/training/simulation/data/no-rotations") + + files_rot = readdir(dir1); + files_norot = readdir(dir2); + + filter!(startswith("timetable_simulation"), files_rot); + filter!(startswith("timetable_simulation"), files_norot); + if isempty(files_rot) || isempty(files_norot) + @error "Files not found"; + exit(1); + end + + delayList_rot = Int[]; + for file in files_rot + + @info "Processing file $dir1/$file"; + + df = CSV.read("$dir1/$file", DataFrame); + + gd = groupby(df, :trainid); + + for g in gd + gs = sort(g, :t_real); + delay = gs[end, :t_real] - gs[end, :t_scheduled]; + push!(delayList_rot, delay); + end + end + + delayList_norot = Int[]; + for file in files_norot + + @info "Processing file $dir2/$file"; + + df = CSV.read("$dir2/$file", DataFrame); + + gd = groupby(df, :trainid); + + for g in gd + gs = sort(g, :t_real); + delay = gs[end, :t_real] - gs[end, :t_scheduled]; + push!(delayList_norot, delay); + end + end + + @info "Building the cumulative distribution"; + + (del_rot, r_rot) = cumul!(delayList_rot); + (del_norot, r_norot) = cumul!(delayList_norot); + + dfout = DataFrame(delay=del_rot, cumulprob=r_rot); + CSV.write("final_dest_delays_rotations.csv", dfout); + + dfout = DataFrame(delay=del_norot, cumulprob=r_norot); + CSV.write("final_dest_delays_norotations.csv", dfout); + + @info "Building the figure"; + plot(del_rot, r_rot, + xscale=:log10, + yscale=:log10, + xguide="delay (s)", + yguide="cumulative probability", + label="with rotations", + lw=4 + # xlims=(10,9000), + # ylims=(1,1000) + ); + plot!(del_norot, r_norot, + label="no rotations", + lw = 3 + ); + @info "\tand saving it." + savefig("figure.pdf"); +end + +main() diff --git a/preprocessing/Manifest.toml b/preprocessing/Manifest.toml index 922abeb..126fee4 100644 --- a/preprocessing/Manifest.toml +++ b/preprocessing/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.8.3" +julia_version = "1.9.2" manifest_format = "2.0" project_hash = "ac193c920ee1d6a7b8f8883c76ffb9c3b0bd1fa8" @@ -15,27 +15,31 @@ uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" [[deps.CSV]] -deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "SentinelArrays", "SnoopPrecompile", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] -git-tree-sha1 = "8c73e96bd6817c2597cfd5615b91fca5deccf1af" +deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] +git-tree-sha1 = "44dbf560808d49041989b8a96cae4cffbeb7966a" uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" -version = "0.10.8" +version = "0.10.11" [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da" +git-tree-sha1 = "9c209fb7536406834aa938fb149964b985de6c83" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.0" +version = "0.7.1" [[deps.Compat]] -deps = ["Dates", "LinearAlgebra", "UUIDs"] -git-tree-sha1 = "00a2cccc7f098ff3b66806862d275ca3db9e6e5a" +deps = ["UUIDs"] +git-tree-sha1 = "4e88377ae7ebeaf29a047aa1ee40826e0b708a5d" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.5.0" +version = "4.7.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "0.5.2+0" +version = "1.0.5+0" [[deps.Crayons]] git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" @@ -43,21 +47,21 @@ uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" version = "4.1.1" [[deps.DataAPI]] -git-tree-sha1 = "e8119c1a33d267e16108be441a287a6981ba1630" +git-tree-sha1 = "8da84edb865b0b5b0100c0666a9bc9a0b71c553c" uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.14.0" +version = "1.15.0" [[deps.DataFrames]] -deps = ["Compat", "DataAPI", "Future", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrettyTables", "Printf", "REPL", "Random", "Reexport", "SnoopPrecompile", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] -git-tree-sha1 = "d4f69885afa5e6149d0cab3818491565cf41446d" +deps = ["Compat", "DataAPI", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "REPL", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "089d29c0fc00a190661517e4f3cba5dcb3fd0c08" uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -version = "1.4.4" +version = "1.6.0" [[deps.DataStructures]] deps = ["Compat", "InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "d1fff3a548102f48987a52a2e0d114fa97d730f0" +git-tree-sha1 = "cf25ccb972fec4e4817764d01c82386ae94f77b4" uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.18.13" +version = "0.18.14" [[deps.DataValueInterfaces]] git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" @@ -94,18 +98,18 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" [[deps.InlineStrings]] deps = ["Parsers"] -git-tree-sha1 = "0cf92ec945125946352f3d46c96976ab972bde6f" +git-tree-sha1 = "9cc2baf75c6d09f9da536ddf58eb2f29dedaf461" uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" -version = "1.3.2" +version = "1.4.0" [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" [[deps.InvertedIndices]] -git-tree-sha1 = "82aec7a3dd64f4d9584659dc0b62ef7db2ef3e19" +git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" -version = "1.2.0" +version = "1.3.0" [[deps.IteratorInterfaceExtensions]] git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" @@ -158,7 +162,7 @@ uuid = "9c8b4983-aa76-5018-a973-4c85ecc9e179" version = "0.9.0" [[deps.LinearAlgebra]] -deps = ["Libdl", "libblastrampoline_jll"] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[deps.Logging]] @@ -171,20 +175,20 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.0+0" +version = "2.28.2+0" [[deps.Missings]] deps = ["DataAPI"] -git-tree-sha1 = "bf210ce90b6c9eed32d25dbcae1ebc565df2687f" +git-tree-sha1 = "f66bdc5de519e8f8ae43bdc598782d35a25b1272" uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" -version = "1.0.2" +version = "1.1.0" [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2022.2.1" +version = "2022.10.11" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" @@ -193,23 +197,23 @@ version = "1.2.0" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.20+0" +version = "0.3.21+4" [[deps.OrderedCollections]] -git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c" +git-tree-sha1 = "d321bf2de576bf25ec4d3e4360faca399afca282" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.4.1" +version = "1.6.0" [[deps.Parsers]] -deps = ["Dates", "SnoopPrecompile"] -git-tree-sha1 = "6466e524967496866901a78fca3f2e9ea445a559" +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "4b2e829ee66d4218e0cef22c0a64ee37cf258c29" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.5.2" +version = "2.7.1" [[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.8.0" +version = "1.9.2" [[deps.PooledArrays]] deps = ["DataAPI", "Future"] @@ -217,17 +221,23 @@ git-tree-sha1 = "a6062fe4063cdafe78f4a0a81cfffb89721b30e7" uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" version = "1.4.2" +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "9673d39decc5feece56ef3940e5dafba15ba0f81" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.1.2" + [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "47e5f437cc0e7ef2ce8406ce1e7e24d44915f88d" +git-tree-sha1 = "7eb1686b4f04b82f96ed7a4ea5890a4f0c7a09f1" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.3.0" +version = "1.4.0" [[deps.PrettyTables]] deps = ["Crayons", "Formatting", "LaTeXStrings", "Markdown", "Reexport", "StringManipulation", "Tables"] -git-tree-sha1 = "96f6db03ab535bdb901300f88335257b0018689d" +git-tree-sha1 = "331cc8048cba270591eab381e7aa3e2e3fef7f5e" uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "2.2.2" +version = "2.2.5" [[deps.Printf]] deps = ["Unicode"] @@ -252,44 +262,45 @@ version = "0.7.0" [[deps.SentinelArrays]] deps = ["Dates", "Random"] -git-tree-sha1 = "efd23b378ea5f2db53a55ae53d3133de4e080aa9" +git-tree-sha1 = "04bdff0b09c65ff3e06a05e3eb7b120223da3d39" uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" -version = "1.3.16" +version = "1.4.0" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -[[deps.SnoopPrecompile]] -git-tree-sha1 = "f604441450a3c0569830946e5b33b78c928e1a85" -uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c" -version = "1.0.1" - [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" [[deps.SortingAlgorithms]] deps = ["DataStructures"] -git-tree-sha1 = "a4ada03f999bd01b3a25dcaa30b2d929fe537e00" +git-tree-sha1 = "c60ec5c62180f27efea3ba2908480f8055e17cee" uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.1.0" +version = "1.1.1" [[deps.SparseArrays]] -deps = ["LinearAlgebra", "Random"] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[deps.Statistics]] deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.9.0" [[deps.StringManipulation]] git-tree-sha1 = "46da2434b41f41ac3594ee9816ce5541c6096123" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" version = "0.3.0" +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "5.10.1+6" + [[deps.TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" -version = "1.0.0" +version = "1.0.3" [[deps.TableTraits]] deps = ["IteratorInterfaceExtensions"] @@ -299,14 +310,14 @@ version = "1.0.1" [[deps.Tables]] deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"] -git-tree-sha1 = "c79322d36826aa2f4fd8ecfa96ddb47b174ac78d" +git-tree-sha1 = "1544b926975372da01227b382066ab70e574a3ec" uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "1.10.0" +version = "1.10.1" [[deps.Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" -version = "1.10.1" +version = "1.10.0" [[deps.Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] @@ -314,9 +325,9 @@ uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [[deps.TranscodingStreams]] deps = ["Random", "Test"] -git-tree-sha1 = "e4bdc63f5c6d62e80eb1c0043fcc0360d5950ff7" +git-tree-sha1 = "9a6ae7ed916312b41236fcef7e0af564ef934769" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.9.10" +version = "0.9.13" [[deps.UUIDs]] deps = ["Random", "SHA"] @@ -338,19 +349,19 @@ version = "1.6.1" [[deps.XML2_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"] -git-tree-sha1 = "58443b63fb7e465a8a7210828c91c08b92132dff" +git-tree-sha1 = "93c41695bc1c08c46c5899f4fe06d6ead504bb73" uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" -version = "2.9.14+0" +version = "2.10.3+0" [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.12+3" +version = "1.2.13+0" [[deps.libblastrampoline_jll]] -deps = ["Artifacts", "Libdl", "OpenBLAS_jll"] +deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.1.1+0" +version = "5.8.0+0" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] diff --git a/preprocessing/PAD-delays_at_final_dest.jl b/preprocessing/PAD-delays_at_final_dest.jl new file mode 100644 index 0000000..202e6a3 --- /dev/null +++ b/preprocessing/PAD-delays_at_final_dest.jl @@ -0,0 +1,72 @@ +using CSV,DataFrames,Plots,Dates; + +function dateToSeconds(d::AbstractString)::Int + dt::DateTime = Dates.DateTime(d, "dd.mm.yyyy HH:MM:SS") + return Int(floor(datetime2unix(dt))) + #return (Dates.hour(dt)*60+Dates.minute(dt))*60+Dates.second(dt) +end + +function string2Date(d::AbstractString) + ds = parse.(Int,split(d,".")); + Date(ds[3]+2000,ds[2],ds[1]) +end + +function cumul!(V::Vector{Int}) + filter!(x->x>0, V); + sort!(V) + r = collect(length(V):-1:1); + r = r./r[1]; + (V, r) +end + +function main(file = "../data/OeBB/2018/PAD-Zuglaufdaten_2018-02.csv") + + df = CSV.read(file, select=[1,4,9,10], DataFrame); + dropmissing!(df); + filter!(x->!isempty(x[3])&&!isempty(x[4]), df); + + select!(df, + 1 => ByRow(string2Date) => :day, + 2 => :trainnr, + # [3,4] => ByRow((x,y)->dateToSeconds(y)-dateToSeconds(x)) => :delay + 3 => ByRow(dateToSeconds) => :t_sched, + 4 => ByRow(dateToSeconds) => :t_real + ); + + filter!(x->month(x.day)==5, df) + + gd = groupby(df, [:day, :trainnr]); + + delayList = Int[]; + + for g in gd + gs = sort(g, :t_real); + delay = gs[end, :t_real] - gs[end, :t_sched]; + push!(delayList, delay); + end + + filter!(x->x>0, delayList); + + del,r = cumul!(delayList); + + dfout = DataFrame(delay=delayList, cumulprob=r); + CSV.write("final_dest_delays_may18.csv", dfout); + + @info "Building the figure"; + plot(del, r, + xscale=:log10, + yscale=:log10, + xguide="delay (s)", + yguide="cumulative probability", + label="May 2018", + lw=4 + # xlims=(10,9000), + # ylims=(1,1000) + ); + + + @info "\tand saving it." + savefig("figure-final_dest_delays_may18.pdf"); +end + +main()