diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c4e712a --- /dev/null +++ b/.gitignore @@ -0,0 +1,332 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +*.mbtiles + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ +**/Properties/launchSettings.json + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..fca3dce --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +# optimize-png-mbtiles + +Walks through all tiles in an mbtiles file (only **raster _PNG_ mbtiles** supported) and optimizes them (_in place!_) using [pngquant](https://pngquant.org/). + +**`optimize-png-mbtiles` has been developed for and tested with [`basemap.at Verwaltungskarte Raster Offline Österreich (bmap_standard_mbtiles_L00bisL16.zip)`](https://www.data.gv.at/katalog/dataset/703fce40-6116-4836-aca4-7dddc33912ab).** + + +If it works with other mbtiles files is a game of luck and depends on their internal database schema and how they were created. +While `basemapat_standard16.MBTiles` adheres to the mbtiles spec from a read-only point of view, its internal database schema is different from the schema of mbtiles created with tools from the Mapbox universe (eg [node-mbtiles](https://github.com/mapbox/node-mbtiles)). + +No analysis of the actual database schema is performed and `optimize-png-mbtiles` most likely will _**not**_ work with mbtiles created with Mapbox tools. + + + +`basemapat_standard16.MBTiles` weighs in at **+16GB** which essentially makes it useless for its intended offline use (think mobile apps or shared hosting). + +Maximum compression reduces those **+16GB to ~4.6GB**, still a lot, but much better to handle. + +![](images/size-comparision.png) + +Maximum compression is of course noticeable in a side by side comparision, but not that bad either: + +original | optimized +---- | ---- +127KB
![](images/10-552-662.png) | 31KB
![](images/10-552-662_optimized.png) +131KB
![](images/10-553-669.png) | 31KB
![](images/10-553-669_optimized.png) +128KB
![](images/10-555-664.png) | 33KB
![](images/10-555-664_optimized.png) +137KB
![](images/10-556-664.png) | 32KB
![](images/10-556-664_optimized.png) + +On my laptop optimizing `basemapat_standard16.MBTiles` takes about: +* **~30 minutes** for tile optimization +* **~3 minutes** for final database vacuum + +![](images/duration.png) + +# Usage + +### Note: +``` +Make sure there's enough free RAM before starting the tool. +At least 5GB (7GB recommended, the more the better). +The final database vacuum needs a lot of it. +``` + +* download `bmap_standard_mbtiles_L00bisL16.zip` from [`basemap.at Verwaltungskarte Raster Offline Österreich`](https://www.data.gv.at/katalog/dataset/703fce40-6116-4836-aca4-7dddc33912ab) +* unzip `bmap_standard_mbtiles_L00bisL16.zip` +* get `optimizepngmbtiles` binary from [`Releases` tab](https://github.com/BergWerkGIS/optimize-png-mbtiles/releases) (sorry, Windows only) +* execute `optimizepngmbtiles.exe -f basemapat_standard16.MBTiles` + +Default settings are optimized for size, sacrificing some quality on the way. + +Adjust settings to get desired output quality. Even moderate compression without visible degradation will save a few GBs already. + +``` +λ optimizepngmbtiles.exe + -f, --mbtiles=VALUE Path to MBTiles + -m, --min-quality=VALUE PNG min quality [0..100]. Default:0 + -x, --max-quality=VALUE PNG max quality [0..100]. Default:20 + -s, --speed=VALUE PNG speed [1..10]. Slower better quality. Default:1 + -t, --threads=VALUE Threads. Default (Processors):8 + -b, --batch-size=VALUE Batch size. Number of tiles processed in one batch. + Default:1000 +``` \ No newline at end of file diff --git a/images/10-552-662.png b/images/10-552-662.png new file mode 100644 index 0000000..2589671 Binary files /dev/null and b/images/10-552-662.png differ diff --git a/images/10-552-662_optimized.png b/images/10-552-662_optimized.png new file mode 100644 index 0000000..44235ec Binary files /dev/null and b/images/10-552-662_optimized.png differ diff --git a/images/10-553-669.png b/images/10-553-669.png new file mode 100644 index 0000000..3ad6e30 Binary files /dev/null and b/images/10-553-669.png differ diff --git a/images/10-553-669_optimized.png b/images/10-553-669_optimized.png new file mode 100644 index 0000000..3730b3d Binary files /dev/null and b/images/10-553-669_optimized.png differ diff --git a/images/10-555-664.png b/images/10-555-664.png new file mode 100644 index 0000000..e299821 Binary files /dev/null and b/images/10-555-664.png differ diff --git a/images/10-555-664_optimized.png b/images/10-555-664_optimized.png new file mode 100644 index 0000000..63995c2 Binary files /dev/null and b/images/10-555-664_optimized.png differ diff --git a/images/10-556-664.png b/images/10-556-664.png new file mode 100644 index 0000000..9807147 Binary files /dev/null and b/images/10-556-664.png differ diff --git a/images/10-556-664_optimized.png b/images/10-556-664_optimized.png new file mode 100644 index 0000000..c6ba5bf Binary files /dev/null and b/images/10-556-664_optimized.png differ diff --git a/images/duration.png b/images/duration.png new file mode 100644 index 0000000..d9d760a Binary files /dev/null and b/images/duration.png differ diff --git a/images/size-comparision.png b/images/size-comparision.png new file mode 100644 index 0000000..5142899 Binary files /dev/null and b/images/size-comparision.png differ diff --git a/optimize-png-mbtiles/Cruncher.cs b/optimize-png-mbtiles/Cruncher.cs new file mode 100644 index 0000000..bab1d8b --- /dev/null +++ b/optimize-png-mbtiles/Cruncher.cs @@ -0,0 +1,244 @@ +using ImageQuant; +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.SQLite; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Threading; +using static System.FormattableString; + + +namespace optimizepngmbtiles { + + + public class Cruncher { + + + private static CruncherSettings _cs; + private static DateTime _dtStart; + private static long _cnter = 0; + private static SQLiteConnection _conn; + + + public Cruncher(CruncherSettings cs) { + _cs = cs; + } + + + public void Go() { + + _dtStart = DateTime.Now; + + try { + + ThreadPool.SetMinThreads(_cs.Threads, _cs.Threads); + ThreadPool.SetMaxThreads(_cs.Threads, _cs.Threads); + + string sqliteFile = Path.GetFullPath(_cs.MbTilesPath); + double sizeStart = getFileSize(sqliteFile); + Console.WriteLine($"sqlite: {sqliteFile}"); + Console.WriteLine(Invariant($"size: {sizeStart:0.00}GB")); + string connStr = $"Data Source={sqliteFile};"; + _conn = new SQLiteConnection(connStr); + _conn.Open(); + + executeCmd("PRAGMA synchronous=OFF"); + executeCmd("PRAGMA count_changes=OFF"); + executeCmd("PRAGMA journal_mode=MEMORY"); + executeCmd("PRAGMA temp_store=MEMORY"); + + List> batches = new List>(); + + using (SQLiteCommand cmd = _conn.CreateCommand()) { + cmd.CommandType = CommandType.Text; + //cmd.CommandText = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles where zoom_level"; + cmd.CommandText = "SELECT zoom_level, tile_column, tile_row FROM tiles where zoom_level"; + using (SQLiteDataReader reader = cmd.ExecuteReader()) { + if (!reader.HasRows) { + Console.WriteLine("MBTiles file does not contain any tiles"); + return; + } + List workItem = new List(_cs.BatchSize); + while (reader.Read()) { + int z = Convert.ToInt32(reader[0]); + long x = Convert.ToInt64(reader[1]); + long y = Convert.ToInt64(reader[2]); + + TileData td = new TileData { + z = z, + x = x, + y = y, + }; + + workItem.Add(td); + + ///////////////////// + /////////////////////// + /////////////// + /// TODO: last batch might get lost + ///////////////////// + ///////////////////// + ///////////////////// + ///////////////////// + + if (workItem.Count == _cs.BatchSize) { + batches.Add(workItem); + workItem = new List(_cs.BatchSize); + } + } + } + } + + + foreach (List workItem in batches) { + ThreadPool.QueueUserWorkItem(crunchTile, new List(workItem)); + } + + + while (ThreadPool.PendingWorkItemCount > 0) { + Thread.Sleep(1000); + } + + + Console.WriteLine($"cnter:{_cnter} CompletedWorkItemCount:{ThreadPool.CompletedWorkItemCount}"); + + + DateTime dtVacStart = DateTime.Now; + try { + Console.WriteLine("VACUUM .... "); + using (SQLiteCommand cmd = _conn.CreateCommand()) { + cmd.CommandType = CommandType.Text; + cmd.CommandText = "VACUUM;"; + cmd.ExecuteNonQuery(); + } + } + catch (Exception ex) { + Console.WriteLine("Error compressing database:"); + Console.WriteLine(ex.ToString()); + } + + DateTime dtStop = DateTime.Now; + TimeSpan spanOverall = dtStop.Subtract(_dtStart); + TimeSpan spanVac = dtStop.Subtract(dtVacStart); + + double sizeDone = getFileSize(sqliteFile); + + Console.WriteLine(""); + Console.WriteLine(Invariant($"SIZE : {sizeStart:0.00}GB => {sizeDone:0.00}GB")); + Console.WriteLine(Invariant($"VACCUMM : {spanVac.Hours}h {spanVac.Minutes}m {spanVac.Seconds}s ({dtVacStart:yyyyMMdd HHmmss} => {dtStop:yyyyMMdd HHmmss})")); + Console.WriteLine(Invariant($"OVERALL : {spanOverall.Hours}h {spanOverall.Minutes}m {spanOverall.Seconds}s ({_dtStart:yyyyMMdd HHmmss} => {dtStop:yyyyMMdd HHmmss})")); + } + catch (Exception ex) { + Console.WriteLine("ERROR:"); + Console.WriteLine(ex.ToString()); + } + finally { + if (null != _conn) { + _conn.Close(); + _conn.Dispose(); + _conn = null; + } + } + } + + + + + private static void crunchTile(object paramData) { + + Thread.CurrentThread.Priority = ThreadPriority.Lowest; + + List tiles = paramData as List; + if (null == tiles) { return; } + + foreach (TileData tile in tiles) { + + // debug message in case of error + string debugQuery = $"SELECT * FROM TILES WHERE zoom_level={tile.z} AND tile_column={tile.x} and tile_row={tile.y}"; + + try { + byte[] data = null; + using (SQLiteCommand cmd = _conn.CreateCommand()) { + cmd.CommandType = CommandType.Text; + cmd.CommandText = $"SELECT tile_data FROM tiles WHERE zoom_level={tile.z} AND tile_column={tile.x} and tile_row={tile.y}"; + using (SQLiteDataReader reader = cmd.ExecuteReader()) { + if (!reader.HasRows) { Console.WriteLine("no data"); return; } + if (!reader.Read()) { return; } + data = (byte[])reader[0]; + } + } + + + //optimize + byte[] optimized = ImageQuantWrapper.Process(data, _cs.MinQuality, _cs.MaxQuality, _cs.Speed); + if (null == optimized) { + Console.WriteLine($"optimization failed, TMS: {debugQuery}"); + return; + } + + // write + using (SQLiteCommand cmdUpdate = _conn.CreateCommand()) { + cmdUpdate.CommandType = CommandType.Text; + cmdUpdate.CommandText = $"UPDATE tiles SET tile_data=@img WHERE zoom_level={tile.z} AND tile_column={tile.x} and tile_row={tile.y}"; + cmdUpdate.Parameters.AddWithValue("@img", optimized); + int rowsAffected = cmdUpdate.ExecuteNonQuery(); + if (1 != rowsAffected) { + Console.WriteLine($"rowsAffected != 1: {debugQuery}"); + return; + } + } + + long cnter = Interlocked.Increment(ref _cnter); + + if (0 == cnter % 5000) { + DateTime dtCurrent = DateTime.Now; + TimeSpan elapsed = dtCurrent.Subtract(_dtStart); + double tps = ((double)cnter) / elapsed.TotalSeconds; + //Console.WriteLine($"{Environment.NewLine}{cnter} {_dtStart:yyyyMMdd HHmmss} => {dtCurrent:yyyyMMdd HHmmss} {elapsed.Hours}h {elapsed.Minutes}m {elapsed.Seconds}s"); + Console.WriteLine(Invariant($"{cnter,6} {elapsed.Hours,2}h {elapsed.Minutes,2}m {elapsed.Seconds,2}s {tps,7:0.0} tiles/second batches left:{ThreadPool.PendingWorkItemCount}")); + } + } + catch (Exception ex) { + Console.WriteLine($"ERROR processing tile {tile}:{Environment.NewLine}{ex.ToString()}"); + } + } + } + + + private double getFileSize(string fileName) { + FileInfo fi = new FileInfo(fileName); + return ((double)fi.Length / 1024.0d / 1024.0d / 1024.0d); + } + + + public static bool executeCmd(string cmdSql) { + + try { + + using (SQLiteCommand cmd = _conn.CreateCommand()) { + + cmd.CommandType = CommandType.Text; + cmd.CommandText = cmdSql.ToString(); + + int rowsAffected = cmd.ExecuteNonQuery(); + } + + return true; + } + catch (Exception ex) { + Console.WriteLine($"ERROR executing database command:{Environment.NewLine}{cmdSql}{Environment.NewLine}{ex}"); + return false; + } + } + + + + + + + + + } +} diff --git a/optimize-png-mbtiles/CruncherSettings.cs b/optimize-png-mbtiles/CruncherSettings.cs new file mode 100644 index 0000000..4362210 --- /dev/null +++ b/optimize-png-mbtiles/CruncherSettings.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace optimizepngmbtiles { + public class CruncherSettings { + + public CruncherSettings() { + //Threads = (int)Math.Floor(((double)Environment.ProcessorCount) * 1.5d); + Threads = Environment.ProcessorCount; + } + + public string MbTilesPath { get; set; } + + public int MinQuality { get; set; } = 0; + public int MaxQuality { get; set; } = 20; + public int Speed { get; set; } = 1; + public int BatchSize { get; set; } = 1000; + public int Threads { get; set; } + + + public override string ToString() { + StringBuilder sb = new StringBuilder(); + + sb.AppendLine($"MbTiles : {MbTilesPath}"); + sb.AppendLine($"Threads : {Threads}"); + sb.AppendLine($"Batch size : {BatchSize}"); + sb.AppendLine($"PNG min quality : {MinQuality}"); + sb.AppendLine($"PNG max quality : {MaxQuality}"); + sb.AppendLine($"PNG speed : {Speed}"); + + return sb.ToString(); + } + + + + + } +} diff --git a/optimize-png-mbtiles/ImageQuant.cs b/optimize-png-mbtiles/ImageQuant.cs new file mode 100644 index 0000000..a2ab92c --- /dev/null +++ b/optimize-png-mbtiles/ImageQuant.cs @@ -0,0 +1,129 @@ +/* +This is an example demonstrating use of libimagequant from C#. +This example code can be freely copied under CC0 (public domain) license. +*/ + +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + + +namespace ImageQuant { + + + using liq_attr_ptr = IntPtr; + using liq_image_ptr = IntPtr; + using liq_result_ptr = IntPtr; + using size_t = UIntPtr; + + + [StructLayout(LayoutKind.Sequential)] + struct liq_color { + public byte r, g, b, a; + }; + + [StructLayout(LayoutKind.Sequential)] + struct liq_palette { + public int count; + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 256)] + public liq_color[] entries; + }; + + enum liq_error { + LIQ_OK = 0, + LIQ_QUALITY_TOO_LOW = 99, + LIQ_VALUE_OUT_OF_RANGE = 100, + LIQ_OUT_OF_MEMORY, + LIQ_ABORTED, + LIQ_BITMAP_NOT_AVAILABLE, + LIQ_BUFFER_TOO_SMALL, + LIQ_INVALID_POINTER, + }; + + + + class imagequant { + + //private const string DLLNAME = @"imagequant.dll"; + //private const string DLLNAME = @"libimagequant.a"; + private const string DLLNAME = "imagequant_sys.dll"; + + [DllImport(DLLNAME)] + public static extern liq_attr_ptr liq_attr_create(); + [DllImport(DLLNAME)] + public static extern liq_attr_ptr liq_attr_copy(liq_attr_ptr attr); + [DllImport(DLLNAME)] + public static extern void liq_attr_destroy(liq_attr_ptr attr); + + [DllImport(DLLNAME)] + public static extern liq_error liq_set_max_colors(liq_attr_ptr attr, int colors); + [DllImport(DLLNAME)] + public static extern int liq_get_max_colors(liq_attr_ptr attr); + [DllImport(DLLNAME)] + public static extern liq_error liq_set_speed(liq_attr_ptr attr, int speed); + [DllImport(DLLNAME)] + public static extern int liq_get_speed(liq_attr_ptr attr); + [DllImport(DLLNAME)] + public static extern liq_error liq_set_min_opacity(liq_attr_ptr attr, int min); + [DllImport(DLLNAME)] + public static extern int liq_get_min_opacity(liq_attr_ptr attr); + [DllImport(DLLNAME)] + public static extern liq_error liq_set_min_posterization(liq_attr_ptr attr, int bits); + [DllImport(DLLNAME)] + public static extern int liq_get_min_posterization(liq_attr_ptr attr); + [DllImport(DLLNAME)] + public static extern liq_error liq_set_quality(liq_attr_ptr attr, int minimum, int maximum); + [DllImport(DLLNAME)] + public static extern int liq_get_min_quality(liq_attr_ptr attr); + [DllImport(DLLNAME)] + public static extern int liq_get_max_quality(liq_attr_ptr attr); + [DllImport(DLLNAME)] + public static extern void liq_set_last_index_transparent(liq_attr_ptr attr, int is_last); + + [DllImport(DLLNAME)] + public static extern liq_image_ptr liq_image_create_rgba(liq_attr_ptr attr, [In, MarshalAs(UnmanagedType.LPArray)] byte[] bitmap, int width, int height, double gamma); + + [DllImport(DLLNAME)] + public static extern liq_error liq_image_set_memory_ownership(liq_image_ptr image, int ownership_flags); + [DllImport(DLLNAME)] + public static extern liq_error liq_image_add_fixed_color(liq_image_ptr img, liq_color color); + [DllImport(DLLNAME)] + public static extern int liq_image_get_width(liq_image_ptr img); + [DllImport(DLLNAME)] + public static extern int liq_image_get_height(liq_image_ptr img); + [DllImport(DLLNAME)] + public static extern void liq_image_destroy(liq_image_ptr img); + + [DllImport(DLLNAME)] + public static extern liq_result_ptr liq_quantize_image(liq_attr_ptr attr, liq_image_ptr input_image); + + [DllImport(DLLNAME)] + public static extern liq_error liq_set_dithering_level(liq_result_ptr res, float dither_level); + [DllImport(DLLNAME)] + public static extern liq_error liq_set_output_gamma(liq_result_ptr res, double gamma); + [DllImport(DLLNAME)] + public static extern double liq_get_output_gamma(liq_result_ptr res); + + [DllImport(DLLNAME)] + public static extern IntPtr liq_get_palette(liq_result_ptr res); + + [DllImport(DLLNAME)] + public static extern liq_error liq_write_remapped_image(liq_result_ptr res, liq_image_ptr input_image, [Out, MarshalAs(UnmanagedType.LPArray)] byte[] buffer, size_t buffer_size); + + [DllImport(DLLNAME)] + public static extern double liq_get_quantization_error(liq_result_ptr res); + [DllImport(DLLNAME)] + public static extern int liq_get_quantization_quality(liq_result_ptr res); + [DllImport(DLLNAME)] + public static extern double liq_get_remapping_error(liq_result_ptr res); + [DllImport(DLLNAME)] + public static extern int liq_get_remapping_quality(liq_result_ptr res); + + [DllImport(DLLNAME)] + public static extern void liq_result_destroy(liq_result_ptr res); + + [DllImport(DLLNAME)] + public static extern int liq_version(); + + } +} \ No newline at end of file diff --git a/optimize-png-mbtiles/ImageQuantWrapper.cs b/optimize-png-mbtiles/ImageQuantWrapper.cs new file mode 100644 index 0000000..e5c4519 --- /dev/null +++ b/optimize-png-mbtiles/ImageQuantWrapper.cs @@ -0,0 +1,237 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Drawing.Imaging; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; + + + +namespace ImageQuant { + + + public sealed class ImageQuantWrapper { + + + public static bool Process(string inFileName, string outFilename, int minQuality, int maxQuality, int speed) { + + try { + + if (File.Exists(outFilename)) { + File.Delete(outFilename); + } + + byte[] bmpIn = File.ReadAllBytes(inFileName); + byte[] bmpOUt = Process(bmpIn, minQuality, maxQuality, speed); + if (null == bmpOUt) { + Console.WriteLine($"Error creating new PNG"); + return false; + } + + File.WriteAllBytes(outFilename, bmpOUt); + + return true; + } + catch (Exception ex) { + Console.WriteLine($"unexpected error: {ex}"); + return false; + } + } + + + /// + /// Returns PNG bytes + /// + /// + /// + public static byte[] Process(byte[] bytesPngIn, int minQuality, int maxQuality, int speed) { + + try { + + byte[] origRaw = null; + int width = -1; + int height = -1; + + using (MemoryStream ms = new MemoryStream(bytesPngIn)) { + using (Bitmap bmp = (Bitmap)Bitmap.FromStream(ms)) { + width = bmp.Width; + height = bmp.Height; + BitmapData bmpData = bmp.LockBits( + new Rectangle(0, 0, bmp.Width, bmp.Height) + , ImageLockMode.ReadOnly + , PixelFormat.Format32bppArgb + ); + long bmpDataLength = bmpData.Stride * bmpData.Height; + origRaw = new byte[bmpDataLength]; + Marshal.Copy(bmpData.Scan0, origRaw, 0, origRaw.Length); + bmp.UnlockBits(bmpData); + } + } + + + //Stride is in BGRA + // we need RGBA!!!!!!!! + for (int i = 2; i < origRaw.Length; i += 4) { + byte tmp = origRaw[i]; + origRaw[i] = origRaw[i - 2]; + origRaw[i - 2] = tmp; + } + + + byte[] bmpOut = ImageQuantWrapper.Compress(origRaw, width, height, minQuality: minQuality, maxQuality: maxQuality, speed: speed); + if (null == bmpOut) { + Console.WriteLine($"Error creating new PNG"); + return null; + } + + return bmpOut; + } + catch (Exception ex) { + Console.WriteLine($"unexpected error: {ex}"); + return null; + } + } + + + + /// + /// Process RAW!!!! bytes + /// + /// RAW BYTES!!!!! NOT(!) Bitmap.Save() bytes!!!! + /// + /// + /// + /// + /// + /// 1: slowest-beast quality, 10: fastest and rough + /// PNG bytes!!!!! + public static byte[] Compress( + byte[] bytesOrig + , int width + , int height + , double gamma = 0.0d + , int minQuality = 50 + , int maxQuality = 90 + , int speed = 1 + ) { + + IntPtr ptrAttr = IntPtr.Zero; + IntPtr ptrImgSrc = IntPtr.Zero; + IntPtr ptrImgResult = IntPtr.Zero; + + try { + + ptrAttr = imagequant.liq_attr_create(); + if (ptrAttr == IntPtr.Zero) { + Console.WriteLine("can't create attr"); + return null; + } + + ptrImgSrc = imagequant.liq_image_create_rgba(ptrAttr, bytesOrig, width, height, gamma); + if (ptrImgSrc == IntPtr.Zero) { + Console.WriteLine("can't create image"); + return null; + } + + var errQual = imagequant.liq_set_quality(ptrAttr, minQuality, maxQuality); + if (liq_error.LIQ_OK != errQual) { + Console.WriteLine("can't set quality"); + return null; + } + var errSpeed = imagequant.liq_set_speed(ptrAttr, speed); + if (liq_error.LIQ_OK != errSpeed) { + Console.WriteLine("can't set speed"); + return null; + } + + ptrImgResult = imagequant.liq_quantize_image(ptrAttr, ptrImgSrc); + if (ptrImgResult == IntPtr.Zero) { + Console.WriteLine("can't quantize image"); + return null; + } + + //var buffer_size = width * height; + /// !!!!! 4x for ARGB + //var buffer_size = width * height * 4; + var buffer_size = bytesOrig.Length; + var bytesRemapped = new byte[buffer_size]; + + var err = imagequant.liq_write_remapped_image(ptrImgResult, ptrImgSrc, bytesRemapped, (UIntPtr)buffer_size); + if (err != liq_error.LIQ_OK) { + Console.WriteLine("remapping error"); + return null; + } + + + // APPLY PALETTE + liq_palette liqPal = (liq_palette)Marshal.PtrToStructure(imagequant.liq_get_palette(ptrImgResult), typeof(liq_palette)); + + byte[] bytesOut = null; + using (Bitmap bmpOut = new Bitmap(width, height, PixelFormat.Format8bppIndexed)) { + + ColorPalette pal = bmpOut.Palette; + + //make sure that we only use as many entries as we have available + int liqPalCnt = liqPal.count; + int bmpPalCnt = pal.Entries.Length; + int palCnt = liqPalCnt < bmpPalCnt ? liqPalCnt : bmpPalCnt; + + for (int i = 0; i < palCnt; i++) { + liq_color liqCol = liqPal.entries[i]; + pal.Entries[i] = Color.FromArgb(liqCol.a, liqCol.r, liqCol.g, liqCol.b); + } + + // if there are more bmp entries than liq entries, set to invisible + if (liqPalCnt < bmpPalCnt) { + for (int i = liqPalCnt; i < bmpPalCnt; i++) { + pal.Entries[i] = Color.FromArgb(0, 0, 0, 0); + } + } + + //!!!!! + // Palette IS NOT A REFERENCE!!! HAS TO BE SET AGAIN!!!!! + bmpOut.Palette = pal; + + BitmapData bmpData = bmpOut.LockBits( + new Rectangle(0, 0, bmpOut.Width, bmpOut.Height) + , ImageLockMode.WriteOnly + //, bmpOut.PixelFormat + //, PixelFormat.Format32bppArgb + //, PixelFormat.Format32bppRgb + //, PixelFormat.Format32bppPArgb + , PixelFormat.Format8bppIndexed + ); + int bmpDataLength = bmpData.Stride * bmpData.Height; + Marshal.Copy(bytesRemapped, 0, bmpData.Scan0, bmpDataLength /* compressed.Length*/); + // !!!!! JUST FOR TESTING, write orignal data + //Marshal.Copy(orig, 0, bmpData.Scan0, bmpDataLength /* compressed.Length*/); + bmpOut.UnlockBits(bmpData); + + + using (MemoryStream msOut = new MemoryStream()) { + bmpOut.Save(msOut, ImageFormat.Png); + bytesOut = msOut.GetBuffer(); + } + } + + return bytesOut; + } + catch (Exception ex) { + Console.WriteLine($"unexpected error {ex}"); + return null; + } + finally { + imagequant.liq_image_destroy(ptrImgSrc); + imagequant.liq_result_destroy(ptrImgResult); + imagequant.liq_attr_destroy(ptrAttr); + + } + } + + + + + + } +} diff --git a/optimize-png-mbtiles/Options.cs b/optimize-png-mbtiles/Options.cs new file mode 100644 index 0000000..821937e --- /dev/null +++ b/optimize-png-mbtiles/Options.cs @@ -0,0 +1,2000 @@ +// +// Options.cs +// +// Authors: +// Jonathan Pryor , +// Federico Di Gregorio +// Rolf Bjarne Kvinge +// +// Copyright (C) 2008 Novell (http://www.novell.com) +// Copyright (C) 2009 Federico Di Gregorio. +// Copyright (C) 2012 Xamarin Inc (http://www.xamarin.com) +// Copyright (C) 2017 Microsoft Corporation (http://www.microsoft.com) +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// + +// Compile With: +// mcs -debug+ -r:System.Core Options.cs -o:Mono.Options.dll -t:library +// mcs -debug+ -d:LINQ -r:System.Core Options.cs -o:Mono.Options.dll -t:library +// +// The LINQ version just changes the implementation of +// OptionSet.Parse(IEnumerable), and confers no semantic changes. + +// +// A Getopt::Long-inspired option parsing library for C#. +// +// Mono.Options.OptionSet is built upon a key/value table, where the +// key is a option format string and the value is a delegate that is +// invoked when the format string is matched. +// +// Option format strings: +// Regex-like BNF Grammar: +// name: .+ +// type: [=:] +// sep: ( [^{}]+ | '{' .+ '}' )? +// aliases: ( name type sep ) ( '|' name type sep )* +// +// Each '|'-delimited name is an alias for the associated action. If the +// format string ends in a '=', it has a required value. If the format +// string ends in a ':', it has an optional value. If neither '=' or ':' +// is present, no value is supported. `=' or `:' need only be defined on one +// alias, but if they are provided on more than one they must be consistent. +// +// Each alias portion may also end with a "key/value separator", which is used +// to split option values if the option accepts > 1 value. If not specified, +// it defaults to '=' and ':'. If specified, it can be any character except +// '{' and '}' OR the *string* between '{' and '}'. If no separator should be +// used (i.e. the separate values should be distinct arguments), then "{}" +// should be used as the separator. +// +// Options are extracted either from the current option by looking for +// the option name followed by an '=' or ':', or is taken from the +// following option IFF: +// - The current option does not contain a '=' or a ':' +// - The current option requires a value (i.e. not a Option type of ':') +// +// The `name' used in the option format string does NOT include any leading +// option indicator, such as '-', '--', or '/'. All three of these are +// permitted/required on any named option. +// +// Option bundling is permitted so long as: +// - '-' is used to start the option group +// - all of the bundled options are a single character +// - at most one of the bundled options accepts a value, and the value +// provided starts from the next character to the end of the string. +// +// This allows specifying '-a -b -c' as '-abc', and specifying '-D name=value' +// as '-Dname=value'. +// +// Option processing is disabled by specifying "--". All options after "--" +// are returned by OptionSet.Parse() unchanged and unprocessed. +// +// Unprocessed options are returned from OptionSet.Parse(). +// +// Examples: +// int verbose = 0; +// OptionSet p = new OptionSet () +// .Add ("v", v => ++verbose) +// .Add ("name=|value=", v => Console.WriteLine (v)); +// p.Parse (new string[]{"-v", "--v", "/v", "-name=A", "/name", "B", "extra"}); +// +// The above would parse the argument string array, and would invoke the +// lambda expression three times, setting `verbose' to 3 when complete. +// It would also print out "A" and "B" to standard output. +// The returned array would contain the string "extra". +// +// C# 3.0 collection initializers are supported and encouraged: +// var p = new OptionSet () { +// { "h|?|help", v => ShowHelp () }, +// }; +// +// System.ComponentModel.TypeConverter is also supported, allowing the use of +// custom data types in the callback type; TypeConverter.ConvertFromString() +// is used to convert the value option to an instance of the specified +// type: +// +// var p = new OptionSet () { +// { "foo=", (Foo f) => Console.WriteLine (f.ToString ()) }, +// }; +// +// Random other tidbits: +// - Boolean options (those w/o '=' or ':' in the option format string) +// are explicitly enabled if they are followed with '+', and explicitly +// disabled if they are followed with '-': +// string a = null; +// var p = new OptionSet () { +// { "a", s => a = s }, +// }; +// p.Parse (new string[]{"-a"}); // sets v != null +// p.Parse (new string[]{"-a+"}); // sets v != null +// p.Parse (new string[]{"-a-"}); // sets v == null +// + +// +// Mono.Options.CommandSet allows easily having separate commands and +// associated command options, allowing creation of a *suite* along the +// lines of **git**(1), **svn**(1), etc. +// +// CommandSet allows intermixing plain text strings for `--help` output, +// Option values -- as supported by OptionSet -- and Command instances, +// which have a name, optional help text, and an optional OptionSet. +// +// var suite = new CommandSet ("suite-name") { +// // Use strings and option values, as with OptionSet +// "usage: suite-name COMMAND [OPTIONS]+", +// { "v:", "verbosity", (int? v) => Verbosity = v.HasValue ? v.Value : Verbosity+1 }, +// // Commands may also be specified +// new Command ("command-name", "command help") { +// Options = new OptionSet {/*...*/}, +// Run = args => { /*...*/}, +// }, +// new MyCommandSubclass (), +// }; +// return suite.Run (new string[]{...}); +// +// CommandSet provides a `help` command, and forwards `help COMMAND` +// to the registered Command instance by invoking Command.Invoke() +// with `--help` as an option. +// + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.ComponentModel; +using System.Globalization; +using System.IO; +#if PCL +using System.Reflection; +#else +using System.Runtime.Serialization; +using System.Security.Permissions; +#endif +using System.Text; +using System.Text.RegularExpressions; + +#if LINQ +using System.Linq; +#endif + +#if TEST +using NDesk.Options; +#endif + +#if PCL +using MessageLocalizerConverter = System.Func; +#else +using MessageLocalizerConverter = System.Converter; +#endif + +#if NDESK_OPTIONS +namespace NDesk.Options +#else +namespace Mono.Options +#endif +{ + static class StringCoda { + + public static IEnumerable WrappedLines (string self, params int[] widths) + { + IEnumerable w = widths; + return WrappedLines (self, w); + } + + public static IEnumerable WrappedLines (string self, IEnumerable widths) + { + if (widths == null) + throw new ArgumentNullException ("widths"); + return CreateWrappedLinesIterator (self, widths); + } + + private static IEnumerable CreateWrappedLinesIterator (string self, IEnumerable widths) + { + if (string.IsNullOrEmpty (self)) { + yield return string.Empty; + yield break; + } + using (IEnumerator ewidths = widths.GetEnumerator ()) { + bool? hw = null; + int width = GetNextWidth (ewidths, int.MaxValue, ref hw); + int start = 0, end; + do { + end = GetLineEnd (start, width, self); + char c = self [end-1]; + if (char.IsWhiteSpace (c)) + --end; + bool needContinuation = end != self.Length && !IsEolChar (c); + string continuation = ""; + if (needContinuation) { + --end; + continuation = "-"; + } + string line = self.Substring (start, end - start) + continuation; + yield return line; + start = end; + if (char.IsWhiteSpace (c)) + ++start; + width = GetNextWidth (ewidths, width, ref hw); + } while (start < self.Length); + } + } + + private static int GetNextWidth (IEnumerator ewidths, int curWidth, ref bool? eValid) + { + if (!eValid.HasValue || (eValid.HasValue && eValid.Value)) { + curWidth = (eValid = ewidths.MoveNext ()).Value ? ewidths.Current : curWidth; + // '.' is any character, - is for a continuation + const string minWidth = ".-"; + if (curWidth < minWidth.Length) + throw new ArgumentOutOfRangeException ("widths", + string.Format ("Element must be >= {0}, was {1}.", minWidth.Length, curWidth)); + return curWidth; + } + // no more elements, use the last element. + return curWidth; + } + + private static bool IsEolChar (char c) + { + return !char.IsLetterOrDigit (c); + } + + private static int GetLineEnd (int start, int length, string description) + { + int end = System.Math.Min (start + length, description.Length); + int sep = -1; + for (int i = start; i < end; ++i) { + if (description [i] == '\n') + return i+1; + if (IsEolChar (description [i])) + sep = i+1; + } + if (sep == -1 || end == description.Length) + return end; + return sep; + } + } + + public class OptionValueCollection : IList, IList { + + List values = new List (); + OptionContext c; + + internal OptionValueCollection (OptionContext c) + { + this.c = c; + } + + #region ICollection + void ICollection.CopyTo (Array array, int index) {(values as ICollection).CopyTo (array, index);} + bool ICollection.IsSynchronized {get {return (values as ICollection).IsSynchronized;}} + object ICollection.SyncRoot {get {return (values as ICollection).SyncRoot;}} + #endregion + + #region ICollection + public void Add (string item) {values.Add (item);} + public void Clear () {values.Clear ();} + public bool Contains (string item) {return values.Contains (item);} + public void CopyTo (string[] array, int arrayIndex) {values.CopyTo (array, arrayIndex);} + public bool Remove (string item) {return values.Remove (item);} + public int Count {get {return values.Count;}} + public bool IsReadOnly {get {return false;}} + #endregion + + #region IEnumerable + IEnumerator IEnumerable.GetEnumerator () {return values.GetEnumerator ();} + #endregion + + #region IEnumerable + public IEnumerator GetEnumerator () {return values.GetEnumerator ();} + #endregion + + #region IList + int IList.Add (object value) {return (values as IList).Add (value);} + bool IList.Contains (object value) {return (values as IList).Contains (value);} + int IList.IndexOf (object value) {return (values as IList).IndexOf (value);} + void IList.Insert (int index, object value) {(values as IList).Insert (index, value);} + void IList.Remove (object value) {(values as IList).Remove (value);} + void IList.RemoveAt (int index) {(values as IList).RemoveAt (index);} + bool IList.IsFixedSize {get {return false;}} + object IList.this [int index] {get {return this [index];} set {(values as IList)[index] = value;}} + #endregion + + #region IList + public int IndexOf (string item) {return values.IndexOf (item);} + public void Insert (int index, string item) {values.Insert (index, item);} + public void RemoveAt (int index) {values.RemoveAt (index);} + + private void AssertValid (int index) + { + if (c.Option == null) + throw new InvalidOperationException ("OptionContext.Option is null."); + if (index >= c.Option.MaxValueCount) + throw new ArgumentOutOfRangeException ("index"); + if (c.Option.OptionValueType == OptionValueType.Required && + index >= values.Count) + throw new OptionException (string.Format ( + c.OptionSet.MessageLocalizer ("Missing required value for option '{0}'."), c.OptionName), + c.OptionName); + } + + public string this [int index] { + get { + AssertValid (index); + return index >= values.Count ? null : values [index]; + } + set { + values [index] = value; + } + } + #endregion + + public List ToList () + { + return new List (values); + } + + public string[] ToArray () + { + return values.ToArray (); + } + + public override string ToString () + { + return string.Join (", ", values.ToArray ()); + } + } + + public class OptionContext { + private Option option; + private string name; + private int index; + private OptionSet set; + private OptionValueCollection c; + + public OptionContext (OptionSet set) + { + this.set = set; + this.c = new OptionValueCollection (this); + } + + public Option Option { + get {return option;} + set {option = value;} + } + + public string OptionName { + get {return name;} + set {name = value;} + } + + public int OptionIndex { + get {return index;} + set {index = value;} + } + + public OptionSet OptionSet { + get {return set;} + } + + public OptionValueCollection OptionValues { + get {return c;} + } + } + + public enum OptionValueType { + None, + Optional, + Required, + } + + public abstract class Option { + string prototype, description; + string[] names; + OptionValueType type; + int count; + string[] separators; + bool hidden; + + protected Option (string prototype, string description) + : this (prototype, description, 1, false) + { + } + + protected Option (string prototype, string description, int maxValueCount) + : this (prototype, description, maxValueCount, false) + { + } + + protected Option (string prototype, string description, int maxValueCount, bool hidden) + { + if (prototype == null) + throw new ArgumentNullException ("prototype"); + if (prototype.Length == 0) + throw new ArgumentException ("Cannot be the empty string.", "prototype"); + if (maxValueCount < 0) + throw new ArgumentOutOfRangeException ("maxValueCount"); + + this.prototype = prototype; + this.description = description; + this.count = maxValueCount; + this.names = (this is OptionSet.Category) + // append GetHashCode() so that "duplicate" categories have distinct + // names, e.g. adding multiple "" categories should be valid. + ? new[]{prototype + this.GetHashCode ()} + : prototype.Split ('|'); + + if (this is OptionSet.Category || this is CommandOption) + return; + + this.type = ParsePrototype (); + this.hidden = hidden; + + if (this.count == 0 && type != OptionValueType.None) + throw new ArgumentException ( + "Cannot provide maxValueCount of 0 for OptionValueType.Required or " + + "OptionValueType.Optional.", + "maxValueCount"); + if (this.type == OptionValueType.None && maxValueCount > 1) + throw new ArgumentException ( + string.Format ("Cannot provide maxValueCount of {0} for OptionValueType.None.", maxValueCount), + "maxValueCount"); + if (Array.IndexOf (names, "<>") >= 0 && + ((names.Length == 1 && this.type != OptionValueType.None) || + (names.Length > 1 && this.MaxValueCount > 1))) + throw new ArgumentException ( + "The default option handler '<>' cannot require values.", + "prototype"); + } + + public string Prototype {get {return prototype;}} + public string Description {get {return description;}} + public OptionValueType OptionValueType {get {return type;}} + public int MaxValueCount {get {return count;}} + public bool Hidden {get {return hidden;}} + + public string[] GetNames () + { + return (string[]) names.Clone (); + } + + public string[] GetValueSeparators () + { + if (separators == null) + return new string [0]; + return (string[]) separators.Clone (); + } + + protected static T Parse (string value, OptionContext c) + { + Type tt = typeof (T); +#if PCL + TypeInfo ti = tt.GetTypeInfo (); +#else + Type ti = tt; +#endif + bool nullable = + ti.IsValueType && + ti.IsGenericType && + !ti.IsGenericTypeDefinition && + ti.GetGenericTypeDefinition () == typeof (Nullable<>); +#if PCL + Type targetType = nullable ? tt.GenericTypeArguments [0] : tt; +#else + Type targetType = nullable ? tt.GetGenericArguments () [0] : tt; +#endif + T t = default (T); + try { + if (value != null) { +#if PCL + if (targetType.GetTypeInfo ().IsEnum) + t = (T) Enum.Parse (targetType, value, true); + else + t = (T) Convert.ChangeType (value, targetType); +#else + TypeConverter conv = TypeDescriptor.GetConverter (targetType); + t = (T) conv.ConvertFromString (value); +#endif + } + } + catch (Exception e) { + throw new OptionException ( + string.Format ( + c.OptionSet.MessageLocalizer ("Could not convert string `{0}' to type {1} for option `{2}'."), + value, targetType.Name, c.OptionName), + c.OptionName, e); + } + return t; + } + + internal string[] Names {get {return names;}} + internal string[] ValueSeparators {get {return separators;}} + + static readonly char[] NameTerminator = new char[]{'=', ':'}; + + private OptionValueType ParsePrototype () + { + char type = '\0'; + List seps = new List (); + for (int i = 0; i < names.Length; ++i) { + string name = names [i]; + if (name.Length == 0) + throw new ArgumentException ("Empty option names are not supported.", "prototype"); + + int end = name.IndexOfAny (NameTerminator); + if (end == -1) + continue; + names [i] = name.Substring (0, end); + if (type == '\0' || type == name [end]) + type = name [end]; + else + throw new ArgumentException ( + string.Format ("Conflicting option types: '{0}' vs. '{1}'.", type, name [end]), + "prototype"); + AddSeparators (name, end, seps); + } + + if (type == '\0') + return OptionValueType.None; + + if (count <= 1 && seps.Count != 0) + throw new ArgumentException ( + string.Format ("Cannot provide key/value separators for Options taking {0} value(s).", count), + "prototype"); + if (count > 1) { + if (seps.Count == 0) + this.separators = new string[]{":", "="}; + else if (seps.Count == 1 && seps [0].Length == 0) + this.separators = null; + else + this.separators = seps.ToArray (); + } + + return type == '=' ? OptionValueType.Required : OptionValueType.Optional; + } + + private static void AddSeparators (string name, int end, ICollection seps) + { + int start = -1; + for (int i = end+1; i < name.Length; ++i) { + switch (name [i]) { + case '{': + if (start != -1) + throw new ArgumentException ( + string.Format ("Ill-formed name/value separator found in \"{0}\".", name), + "prototype"); + start = i+1; + break; + case '}': + if (start == -1) + throw new ArgumentException ( + string.Format ("Ill-formed name/value separator found in \"{0}\".", name), + "prototype"); + seps.Add (name.Substring (start, i-start)); + start = -1; + break; + default: + if (start == -1) + seps.Add (name [i].ToString ()); + break; + } + } + if (start != -1) + throw new ArgumentException ( + string.Format ("Ill-formed name/value separator found in \"{0}\".", name), + "prototype"); + } + + public void Invoke (OptionContext c) + { + OnParseComplete (c); + c.OptionName = null; + c.Option = null; + c.OptionValues.Clear (); + } + + protected abstract void OnParseComplete (OptionContext c); + + internal void InvokeOnParseComplete (OptionContext c) + { + OnParseComplete (c); + } + + public override string ToString () + { + return Prototype; + } + } + + public abstract class ArgumentSource { + + protected ArgumentSource () + { + } + + public abstract string[] GetNames (); + public abstract string Description { get; } + public abstract bool GetArguments (string value, out IEnumerable replacement); + +#if !PCL || NETSTANDARD1_3 + public static IEnumerable GetArgumentsFromFile (string file) + { + return GetArguments (File.OpenText (file), true); + } +#endif + + public static IEnumerable GetArguments (TextReader reader) + { + return GetArguments (reader, false); + } + + // Cribbed from mcs/driver.cs:LoadArgs(string) + static IEnumerable GetArguments (TextReader reader, bool close) + { + try { + StringBuilder arg = new StringBuilder (); + + string line; + while ((line = reader.ReadLine ()) != null) { + int t = line.Length; + + for (int i = 0; i < t; i++) { + char c = line [i]; + + if (c == '"' || c == '\'') { + char end = c; + + for (i++; i < t; i++){ + c = line [i]; + + if (c == end) + break; + arg.Append (c); + } + } else if (c == ' ') { + if (arg.Length > 0) { + yield return arg.ToString (); + arg.Length = 0; + } + } else + arg.Append (c); + } + if (arg.Length > 0) { + yield return arg.ToString (); + arg.Length = 0; + } + } + } + finally { + if (close) + reader.Dispose (); + } + } + } + +#if !PCL || NETSTANDARD1_3 + public class ResponseFileSource : ArgumentSource { + + public override string[] GetNames () + { + return new string[]{"@file"}; + } + + public override string Description { + get {return "Read response file for more options.";} + } + + public override bool GetArguments (string value, out IEnumerable replacement) + { + if (string.IsNullOrEmpty (value) || !value.StartsWith ("@")) { + replacement = null; + return false; + } + replacement = ArgumentSource.GetArgumentsFromFile (value.Substring (1)); + return true; + } + } +#endif + +#if !PCL + [Serializable] +#endif + public class OptionException : Exception { + private string option; + + public OptionException () + { + } + + public OptionException (string message, string optionName) + : base (message) + { + this.option = optionName; + } + + public OptionException (string message, string optionName, Exception innerException) + : base (message, innerException) + { + this.option = optionName; + } + +#if !PCL + protected OptionException (SerializationInfo info, StreamingContext context) + : base (info, context) + { + this.option = info.GetString ("OptionName"); + } +#endif + + public string OptionName { + get {return this.option;} + } + +#if !PCL +#pragma warning disable 618 // SecurityPermissionAttribute is obsolete + [SecurityPermission (SecurityAction.LinkDemand, SerializationFormatter = true)] +#pragma warning restore 618 + public override void GetObjectData (SerializationInfo info, StreamingContext context) + { + base.GetObjectData (info, context); + info.AddValue ("OptionName", option); + } +#endif + } + + public delegate void OptionAction (TKey key, TValue value); + + public class OptionSet : KeyedCollection + { + public OptionSet () + : this (null) + { + } + + public OptionSet (MessageLocalizerConverter localizer) + { + this.roSources = new ReadOnlyCollection (sources); + this.localizer = localizer; + if (this.localizer == null) { + this.localizer = delegate (string f) { + return f; + }; + } + } + + MessageLocalizerConverter localizer; + + public MessageLocalizerConverter MessageLocalizer { + get {return localizer;} + internal set {localizer = value;} + } + + List sources = new List (); + ReadOnlyCollection roSources; + + public ReadOnlyCollection ArgumentSources { + get {return roSources;} + } + + + protected override string GetKeyForItem (Option item) + { + if (item == null) + throw new ArgumentNullException ("option"); + if (item.Names != null && item.Names.Length > 0) + return item.Names [0]; + // This should never happen, as it's invalid for Option to be + // constructed w/o any names. + throw new InvalidOperationException ("Option has no names!"); + } + + [Obsolete ("Use KeyedCollection.this[string]")] + protected Option GetOptionForName (string option) + { + if (option == null) + throw new ArgumentNullException ("option"); + try { + return base [option]; + } + catch (KeyNotFoundException) { + return null; + } + } + + protected override void InsertItem (int index, Option item) + { + base.InsertItem (index, item); + AddImpl (item); + } + + protected override void RemoveItem (int index) + { + Option p = Items [index]; + base.RemoveItem (index); + // KeyedCollection.RemoveItem() handles the 0th item + for (int i = 1; i < p.Names.Length; ++i) { + Dictionary.Remove (p.Names [i]); + } + } + + protected override void SetItem (int index, Option item) + { + base.SetItem (index, item); + AddImpl (item); + } + + private void AddImpl (Option option) + { + if (option == null) + throw new ArgumentNullException ("option"); + List added = new List (option.Names.Length); + try { + // KeyedCollection.InsertItem/SetItem handle the 0th name. + for (int i = 1; i < option.Names.Length; ++i) { + Dictionary.Add (option.Names [i], option); + added.Add (option.Names [i]); + } + } + catch (Exception) { + foreach (string name in added) + Dictionary.Remove (name); + throw; + } + } + + public OptionSet Add (string header) + { + if (header == null) + throw new ArgumentNullException ("header"); + Add (new Category (header)); + return this; + } + + internal sealed class Category : Option { + + // Prototype starts with '=' because this is an invalid prototype + // (see Option.ParsePrototype(), and thus it'll prevent Category + // instances from being accidentally used as normal options. + public Category (string description) + : base ("=:Category:= " + description, description) + { + } + + protected override void OnParseComplete (OptionContext c) + { + throw new NotSupportedException ("Category.OnParseComplete should not be invoked."); + } + } + + + public new OptionSet Add (Option option) + { + base.Add (option); + return this; + } + + sealed class ActionOption : Option { + Action action; + + public ActionOption (string prototype, string description, int count, Action action) + : this (prototype, description, count, action, false) + { + } + + public ActionOption (string prototype, string description, int count, Action action, bool hidden) + : base (prototype, description, count, hidden) + { + if (action == null) + throw new ArgumentNullException ("action"); + this.action = action; + } + + protected override void OnParseComplete (OptionContext c) + { + action (c.OptionValues); + } + } + + public OptionSet Add (string prototype, Action action) + { + return Add (prototype, null, action); + } + + public OptionSet Add (string prototype, string description, Action action) + { + return Add (prototype, description, action, false); + } + + public OptionSet Add (string prototype, string description, Action action, bool hidden) + { + if (action == null) + throw new ArgumentNullException ("action"); + Option p = new ActionOption (prototype, description, 1, + delegate (OptionValueCollection v) { action (v [0]); }, hidden); + base.Add (p); + return this; + } + + public OptionSet Add (string prototype, OptionAction action) + { + return Add (prototype, null, action); + } + + public OptionSet Add (string prototype, string description, OptionAction action) + { + return Add (prototype, description, action, false); + } + + public OptionSet Add (string prototype, string description, OptionAction action, bool hidden) { + if (action == null) + throw new ArgumentNullException ("action"); + Option p = new ActionOption (prototype, description, 2, + delegate (OptionValueCollection v) {action (v [0], v [1]);}, hidden); + base.Add (p); + return this; + } + + sealed class ActionOption : Option { + Action action; + + public ActionOption (string prototype, string description, Action action) + : base (prototype, description, 1) + { + if (action == null) + throw new ArgumentNullException ("action"); + this.action = action; + } + + protected override void OnParseComplete (OptionContext c) + { + action (Parse (c.OptionValues [0], c)); + } + } + + sealed class ActionOption : Option { + OptionAction action; + + public ActionOption (string prototype, string description, OptionAction action) + : base (prototype, description, 2) + { + if (action == null) + throw new ArgumentNullException ("action"); + this.action = action; + } + + protected override void OnParseComplete (OptionContext c) + { + action ( + Parse (c.OptionValues [0], c), + Parse (c.OptionValues [1], c)); + } + } + + public OptionSet Add (string prototype, Action action) + { + return Add (prototype, null, action); + } + + public OptionSet Add (string prototype, string description, Action action) + { + return Add (new ActionOption (prototype, description, action)); + } + + public OptionSet Add (string prototype, OptionAction action) + { + return Add (prototype, null, action); + } + + public OptionSet Add (string prototype, string description, OptionAction action) + { + return Add (new ActionOption (prototype, description, action)); + } + + public OptionSet Add (ArgumentSource source) + { + if (source == null) + throw new ArgumentNullException ("source"); + sources.Add (source); + return this; + } + + protected virtual OptionContext CreateOptionContext () + { + return new OptionContext (this); + } + + public List Parse (IEnumerable arguments) + { + if (arguments == null) + throw new ArgumentNullException ("arguments"); + OptionContext c = CreateOptionContext (); + c.OptionIndex = -1; + bool process = true; + List unprocessed = new List (); + Option def = Contains ("<>") ? this ["<>"] : null; + ArgumentEnumerator ae = new ArgumentEnumerator (arguments); + foreach (string argument in ae) { + ++c.OptionIndex; + if (argument == "--") { + process = false; + continue; + } + if (!process) { + Unprocessed (unprocessed, def, c, argument); + continue; + } + if (AddSource (ae, argument)) + continue; + if (!Parse (argument, c)) + Unprocessed (unprocessed, def, c, argument); + } + if (c.Option != null) + c.Option.Invoke (c); + return unprocessed; + } + + class ArgumentEnumerator : IEnumerable { + List> sources = new List> (); + + public ArgumentEnumerator (IEnumerable arguments) + { + sources.Add (arguments.GetEnumerator ()); + } + + public void Add (IEnumerable arguments) + { + sources.Add (arguments.GetEnumerator ()); + } + + public IEnumerator GetEnumerator () + { + do { + IEnumerator c = sources [sources.Count-1]; + if (c.MoveNext ()) + yield return c.Current; + else { + c.Dispose (); + sources.RemoveAt (sources.Count-1); + } + } while (sources.Count > 0); + } + + IEnumerator IEnumerable.GetEnumerator () + { + return GetEnumerator (); + } + } + + bool AddSource (ArgumentEnumerator ae, string argument) + { + foreach (ArgumentSource source in sources) { + IEnumerable replacement; + if (!source.GetArguments (argument, out replacement)) + continue; + ae.Add (replacement); + return true; + } + return false; + } + + private static bool Unprocessed (ICollection extra, Option def, OptionContext c, string argument) + { + if (def == null) { + extra.Add (argument); + return false; + } + c.OptionValues.Add (argument); + c.Option = def; + c.Option.Invoke (c); + return false; + } + + private readonly Regex ValueOption = new Regex ( + @"^(?--|-|/)(?[^:=]+)((?[:=])(?.*))?$"); + + protected bool GetOptionParts (string argument, out string flag, out string name, out string sep, out string value) + { + if (argument == null) + throw new ArgumentNullException ("argument"); + + flag = name = sep = value = null; + Match m = ValueOption.Match (argument); + if (!m.Success) { + return false; + } + flag = m.Groups ["flag"].Value; + name = m.Groups ["name"].Value; + if (m.Groups ["sep"].Success && m.Groups ["value"].Success) { + sep = m.Groups ["sep"].Value; + value = m.Groups ["value"].Value; + } + return true; + } + + protected virtual bool Parse (string argument, OptionContext c) + { + if (c.Option != null) { + ParseValue (argument, c); + return true; + } + + string f, n, s, v; + if (!GetOptionParts (argument, out f, out n, out s, out v)) + return false; + + Option p; + if (Contains (n)) { + p = this [n]; + c.OptionName = f + n; + c.Option = p; + switch (p.OptionValueType) { + case OptionValueType.None: + c.OptionValues.Add (n); + c.Option.Invoke (c); + break; + case OptionValueType.Optional: + case OptionValueType.Required: + ParseValue (v, c); + break; + } + return true; + } + // no match; is it a bool option? + if (ParseBool (argument, n, c)) + return true; + // is it a bundled option? + if (ParseBundledValue (f, string.Concat (n + s + v), c)) + return true; + + return false; + } + + private void ParseValue (string option, OptionContext c) + { + if (option != null) + foreach (string o in c.Option.ValueSeparators != null + ? option.Split (c.Option.ValueSeparators, c.Option.MaxValueCount - c.OptionValues.Count, StringSplitOptions.None) + : new string[]{option}) { + c.OptionValues.Add (o); + } + if (c.OptionValues.Count == c.Option.MaxValueCount || + c.Option.OptionValueType == OptionValueType.Optional) + c.Option.Invoke (c); + else if (c.OptionValues.Count > c.Option.MaxValueCount) { + throw new OptionException (localizer (string.Format ( + "Error: Found {0} option values when expecting {1}.", + c.OptionValues.Count, c.Option.MaxValueCount)), + c.OptionName); + } + } + + private bool ParseBool (string option, string n, OptionContext c) + { + Option p; + string rn; + if (n.Length >= 1 && (n [n.Length-1] == '+' || n [n.Length-1] == '-') && + Contains ((rn = n.Substring (0, n.Length-1)))) { + p = this [rn]; + string v = n [n.Length-1] == '+' ? option : null; + c.OptionName = option; + c.Option = p; + c.OptionValues.Add (v); + p.Invoke (c); + return true; + } + return false; + } + + private bool ParseBundledValue (string f, string n, OptionContext c) + { + if (f != "-") + return false; + for (int i = 0; i < n.Length; ++i) { + Option p; + string opt = f + n [i].ToString (); + string rn = n [i].ToString (); + if (!Contains (rn)) { + if (i == 0) + return false; + throw new OptionException (string.Format (localizer ( + "Cannot use unregistered option '{0}' in bundle '{1}'."), rn, f + n), null); + } + p = this [rn]; + switch (p.OptionValueType) { + case OptionValueType.None: + Invoke (c, opt, n, p); + break; + case OptionValueType.Optional: + case OptionValueType.Required: { + string v = n.Substring (i+1); + c.Option = p; + c.OptionName = opt; + ParseValue (v.Length != 0 ? v : null, c); + return true; + } + default: + throw new InvalidOperationException ("Unknown OptionValueType: " + p.OptionValueType); + } + } + return true; + } + + private static void Invoke (OptionContext c, string name, string value, Option option) + { + c.OptionName = name; + c.Option = option; + c.OptionValues.Add (value); + option.Invoke (c); + } + + private const int OptionWidth = 29; + private const int Description_FirstWidth = 80 - OptionWidth; + private const int Description_RemWidth = 80 - OptionWidth - 2; + + static readonly string CommandHelpIndentStart = new string (' ', OptionWidth); + static readonly string CommandHelpIndentRemaining = new string (' ', OptionWidth + 2); + + public void WriteOptionDescriptions (TextWriter o) + { + foreach (Option p in this) { + int written = 0; + + if (p.Hidden) + continue; + + Category c = p as Category; + if (c != null) { + WriteDescription (o, p.Description, "", 80, 80); + continue; + } + CommandOption co = p as CommandOption; + if (co != null) { + WriteCommandDescription (o, co.Command, co.CommandName); + continue; + } + + if (!WriteOptionPrototype (o, p, ref written)) + continue; + + if (written < OptionWidth) + o.Write (new string (' ', OptionWidth - written)); + else { + o.WriteLine (); + o.Write (new string (' ', OptionWidth)); + } + + WriteDescription (o, p.Description, new string (' ', OptionWidth+2), + Description_FirstWidth, Description_RemWidth); + } + + foreach (ArgumentSource s in sources) { + string[] names = s.GetNames (); + if (names == null || names.Length == 0) + continue; + + int written = 0; + + Write (o, ref written, " "); + Write (o, ref written, names [0]); + for (int i = 1; i < names.Length; ++i) { + Write (o, ref written, ", "); + Write (o, ref written, names [i]); + } + + if (written < OptionWidth) + o.Write (new string (' ', OptionWidth - written)); + else { + o.WriteLine (); + o.Write (new string (' ', OptionWidth)); + } + + WriteDescription (o, s.Description, new string (' ', OptionWidth+2), + Description_FirstWidth, Description_RemWidth); + } + } + + internal void WriteCommandDescription (TextWriter o, Command c, string commandName) + { + var name = new string (' ', 8) + (commandName ?? c.Name); + if (name.Length < OptionWidth - 1) { + WriteDescription (o, name + new string (' ', OptionWidth - name.Length) + c.Help, CommandHelpIndentRemaining, 80, Description_RemWidth); + } else { + WriteDescription (o, name, "", 80, 80); + WriteDescription (o, CommandHelpIndentStart + c.Help, CommandHelpIndentRemaining, 80, Description_RemWidth); + } + } + + void WriteDescription (TextWriter o, string value, string prefix, int firstWidth, int remWidth) + { + bool indent = false; + foreach (string line in GetLines (localizer (GetDescription (value)), firstWidth, remWidth)) { + if (indent) + o.Write (prefix); + o.WriteLine (line); + indent = true; + } + } + + bool WriteOptionPrototype (TextWriter o, Option p, ref int written) + { + string[] names = p.Names; + + int i = GetNextOptionIndex (names, 0); + if (i == names.Length) + return false; + + if (names [i].Length == 1) { + Write (o, ref written, " -"); + Write (o, ref written, names [0]); + } + else { + Write (o, ref written, " --"); + Write (o, ref written, names [0]); + } + + for ( i = GetNextOptionIndex (names, i+1); + i < names.Length; i = GetNextOptionIndex (names, i+1)) { + Write (o, ref written, ", "); + Write (o, ref written, names [i].Length == 1 ? "-" : "--"); + Write (o, ref written, names [i]); + } + + if (p.OptionValueType == OptionValueType.Optional || + p.OptionValueType == OptionValueType.Required) { + if (p.OptionValueType == OptionValueType.Optional) { + Write (o, ref written, localizer ("[")); + } + Write (o, ref written, localizer ("=" + GetArgumentName (0, p.MaxValueCount, p.Description))); + string sep = p.ValueSeparators != null && p.ValueSeparators.Length > 0 + ? p.ValueSeparators [0] + : " "; + for (int c = 1; c < p.MaxValueCount; ++c) { + Write (o, ref written, localizer (sep + GetArgumentName (c, p.MaxValueCount, p.Description))); + } + if (p.OptionValueType == OptionValueType.Optional) { + Write (o, ref written, localizer ("]")); + } + } + return true; + } + + static int GetNextOptionIndex (string[] names, int i) + { + while (i < names.Length && names [i] == "<>") { + ++i; + } + return i; + } + + static void Write (TextWriter o, ref int n, string s) + { + n += s.Length; + o.Write (s); + } + + static string GetArgumentName (int index, int maxIndex, string description) + { + var matches = Regex.Matches (description ?? "", @"(?<=(? 1 + if (maxIndex > 1 && parts.Length == 2 && + parts[0] == index.ToString (CultureInfo.InvariantCulture)) { + argName = parts[1]; + } + } + + if (string.IsNullOrEmpty (argName)) { + argName = maxIndex == 1 ? "VALUE" : "VALUE" + (index + 1); + } + return argName; + } + + private static string GetDescription (string description) + { + if (description == null) + return string.Empty; + StringBuilder sb = new StringBuilder (description.Length); + int start = -1; + for (int i = 0; i < description.Length; ++i) { + switch (description [i]) { + case '{': + if (i == start) { + sb.Append ('{'); + start = -1; + } + else if (start < 0) + start = i + 1; + break; + case '}': + if (start < 0) { + if ((i+1) == description.Length || description [i+1] != '}') + throw new InvalidOperationException ("Invalid option description: " + description); + ++i; + sb.Append ("}"); + } + else { + sb.Append (description.Substring (start, i - start)); + start = -1; + } + break; + case ':': + if (start < 0) + goto default; + start = i + 1; + break; + default: + if (start < 0) + sb.Append (description [i]); + break; + } + } + return sb.ToString (); + } + + private static IEnumerable GetLines (string description, int firstWidth, int remWidth) + { + return StringCoda.WrappedLines (description, firstWidth, remWidth); + } + } + + public class Command + { + public string Name {get;} + public string Help {get;} + + public OptionSet Options {get; set;} + public Action> Run {get; set;} + + public CommandSet CommandSet {get; internal set;} + + public Command (string name, string help = null) + { + if (string.IsNullOrEmpty (name)) + throw new ArgumentNullException (nameof (name)); + + Name = NormalizeCommandName (name); + Help = help; + } + + static string NormalizeCommandName (string name) + { + var value = new StringBuilder (name.Length); + var space = false; + for (int i = 0; i < name.Length; ++i) { + if (!char.IsWhiteSpace (name, i)) { + space = false; + value.Append (name [i]); + } + else if (!space) { + space = true; + value.Append (' '); + } + } + return value.ToString (); + } + + public virtual int Invoke (IEnumerable arguments) + { + var rest = Options?.Parse (arguments) ?? arguments; + Run?.Invoke (rest); + return 0; + } + } + + class CommandOption : Option + { + public Command Command {get;} + public string CommandName {get;} + + // Prototype starts with '=' because this is an invalid prototype + // (see Option.ParsePrototype(), and thus it'll prevent Category + // instances from being accidentally used as normal options. + public CommandOption (Command command, string commandName = null, bool hidden = false) + : base ("=:Command:= " + (commandName ?? command?.Name), (commandName ?? command?.Name), maxValueCount: 0, hidden: hidden) + { + if (command == null) + throw new ArgumentNullException (nameof (command)); + Command = command; + CommandName = commandName ?? command.Name; + } + + protected override void OnParseComplete (OptionContext c) + { + throw new NotSupportedException ("CommandOption.OnParseComplete should not be invoked."); + } + } + + class HelpOption : Option + { + Option option; + CommandSet commands; + + public HelpOption (CommandSet commands, Option d) + : base (d.Prototype, d.Description, d.MaxValueCount, d.Hidden) + { + this.commands = commands; + this.option = d; + } + + protected override void OnParseComplete (OptionContext c) + { + commands.showHelp = true; + + option?.InvokeOnParseComplete (c); + } + } + + class CommandOptionSet : OptionSet + { + CommandSet commands; + + public CommandOptionSet (CommandSet commands, MessageLocalizerConverter localizer) + : base (localizer) + { + this.commands = commands; + } + + protected override void SetItem (int index, Option item) + { + if (ShouldWrapOption (item)) { + base.SetItem (index, new HelpOption (commands, item)); + return; + } + base.SetItem (index, item); + } + + bool ShouldWrapOption (Option item) + { + if (item == null) + return false; + var help = item as HelpOption; + if (help != null) + return false; + foreach (var n in item.Names) { + if (n == "help") + return true; + } + return false; + } + + protected override void InsertItem (int index, Option item) + { + if (ShouldWrapOption (item)) { + base.InsertItem (index, new HelpOption (commands, item)); + return; + } + base.InsertItem (index, item); + } + } + + public class CommandSet : KeyedCollection + { + readonly string suite; + + OptionSet options; + TextWriter outWriter; + TextWriter errorWriter; + + internal List NestedCommandSets; + + internal HelpCommand help; + + internal bool showHelp; + + internal OptionSet Options => options; + +#if !PCL || NETSTANDARD1_3 + public CommandSet(string suite, MessageLocalizerConverter localizer = null) + : this(suite, Console.Out, Console.Error, localizer) + { + } +#endif + + public CommandSet (string suite, TextWriter output, TextWriter error, MessageLocalizerConverter localizer = null) + { + if (suite == null) + throw new ArgumentNullException (nameof (suite)); + if (output == null) + throw new ArgumentNullException (nameof (output)); + if (error == null) + throw new ArgumentNullException (nameof (error)); + + this.suite = suite; + options = new CommandOptionSet (this, localizer); + outWriter = output; + errorWriter = error; + } + + public string Suite => suite; + public TextWriter Out => outWriter; + public TextWriter Error => errorWriter; + public MessageLocalizerConverter MessageLocalizer => options.MessageLocalizer; + + protected override string GetKeyForItem (Command item) + { + return item?.Name; + } + + public new CommandSet Add (Command value) + { + if (value == null) + throw new ArgumentNullException (nameof (value)); + AddCommand (value); + options.Add (new CommandOption (value)); + return this; + } + + void AddCommand (Command value) + { + if (value.CommandSet != null && value.CommandSet != this) { + throw new ArgumentException ("Command instances can only be added to a single CommandSet.", nameof (value)); + } + value.CommandSet = this; + if (value.Options != null) { + value.Options.MessageLocalizer = options.MessageLocalizer; + } + + base.Add (value); + + help = help ?? value as HelpCommand; + } + + public CommandSet Add (string header) + { + options.Add (header); + return this; + } + + public CommandSet Add (Option option) + { + options.Add (option); + return this; + } + + public CommandSet Add (string prototype, Action action) + { + options.Add (prototype, action); + return this; + } + + public CommandSet Add (string prototype, string description, Action action) + { + options.Add (prototype, description, action); + return this; + } + + public CommandSet Add (string prototype, string description, Action action, bool hidden) + { + options.Add (prototype, description, action, hidden); + return this; + } + + public CommandSet Add (string prototype, OptionAction action) + { + options.Add (prototype, action); + return this; + } + + public CommandSet Add (string prototype, string description, OptionAction action) + { + options.Add (prototype, description, action); + return this; + } + + public CommandSet Add (string prototype, string description, OptionAction action, bool hidden) + { + options.Add (prototype, description, action, hidden); + return this; + } + + public CommandSet Add (string prototype, Action action) + { + options.Add (prototype, null, action); + return this; + } + + public CommandSet Add (string prototype, string description, Action action) + { + options.Add (prototype, description, action); + return this; + } + + public CommandSet Add (string prototype, OptionAction action) + { + options.Add (prototype, action); + return this; + } + + public CommandSet Add (string prototype, string description, OptionAction action) + { + options.Add (prototype, description, action); + return this; + } + + public CommandSet Add (ArgumentSource source) + { + options.Add (source); + return this; + } + + public CommandSet Add (CommandSet nestedCommands) + { + if (nestedCommands == null) + throw new ArgumentNullException (nameof (nestedCommands)); + + if (NestedCommandSets == null) { + NestedCommandSets = new List (); + } + + if (!AlreadyAdded (nestedCommands)) { + NestedCommandSets.Add (nestedCommands); + foreach (var o in nestedCommands.options) { + if (o is CommandOption c) { + options.Add (new CommandOption (c.Command, $"{nestedCommands.Suite} {c.CommandName}")); + } + else { + options.Add (o); + } + } + } + + nestedCommands.options = this.options; + nestedCommands.outWriter = this.outWriter; + nestedCommands.errorWriter = this.errorWriter; + + return this; + } + + bool AlreadyAdded (CommandSet value) + { + if (value == this) + return true; + if (NestedCommandSets == null) + return false; + foreach (var nc in NestedCommandSets) { + if (nc.AlreadyAdded (value)) + return true; + } + return false; + } + + public IEnumerable GetCompletions (string prefix = null) + { + string rest; + ExtractToken (ref prefix, out rest); + + foreach (var command in this) { + if (command.Name.StartsWith (prefix, StringComparison.OrdinalIgnoreCase)) { + yield return command.Name; + } + } + + if (NestedCommandSets == null) + yield break; + + foreach (var subset in NestedCommandSets) { + if (subset.Suite.StartsWith (prefix, StringComparison.OrdinalIgnoreCase)) { + foreach (var c in subset.GetCompletions (rest)) { + yield return $"{subset.Suite} {c}"; + } + } + } + } + + static void ExtractToken (ref string input, out string rest) + { + rest = ""; + input = input ?? ""; + + int top = input.Length; + for (int i = 0; i < top; i++) { + if (char.IsWhiteSpace (input [i])) + continue; + + for (int j = i; j < top; j++) { + if (char.IsWhiteSpace (input [j])) { + rest = input.Substring (j).Trim (); + input = input.Substring (i, j).Trim (); + return; + } + } + rest = ""; + if (i != 0) + input = input.Substring (i).Trim (); + return; + } + } + + public int Run (IEnumerable arguments) + { + if (arguments == null) + throw new ArgumentNullException (nameof (arguments)); + + this.showHelp = false; + if (help == null) { + help = new HelpCommand (); + AddCommand (help); + } + Action setHelp = v => showHelp = v != null; + if (!options.Contains ("help")) { + options.Add ("help", "", setHelp, hidden: true); + } + if (!options.Contains ("?")) { + options.Add ("?", "", setHelp, hidden: true); + } + var extra = options.Parse (arguments); + if (extra.Count == 0) { + if (showHelp) { + return help.Invoke (extra); + } + Out.WriteLine (options.MessageLocalizer ($"Use `{Suite} help` for usage.")); + return 1; + } + var command = GetCommand (extra); + if (command == null) { + help.WriteUnknownCommand (extra [0]); + return 1; + } + if (showHelp) { + if (command.Options?.Contains ("help") ?? true) { + extra.Add ("--help"); + return command.Invoke (extra); + } + command.Options.WriteOptionDescriptions (Out); + return 0; + } + return command.Invoke (extra); + } + + internal Command GetCommand (List extra) + { + return TryGetLocalCommand (extra) ?? TryGetNestedCommand (extra); + } + + Command TryGetLocalCommand (List extra) + { + var name = extra [0]; + if (Contains (name)) { + extra.RemoveAt (0); + return this [name]; + } + for (int i = 1; i < extra.Count; ++i) { + name = name + " " + extra [i]; + if (!Contains (name)) + continue; + extra.RemoveRange (0, i+1); + return this [name]; + } + return null; + } + + Command TryGetNestedCommand (List extra) + { + if (NestedCommandSets == null) + return null; + + var nestedCommands = NestedCommandSets.Find (c => c.Suite == extra [0]); + if (nestedCommands == null) + return null; + + var extraCopy = new List (extra); + extraCopy.RemoveAt (0); + if (extraCopy.Count == 0) + return null; + + var command = nestedCommands.GetCommand (extraCopy); + if (command != null) { + extra.Clear (); + extra.AddRange (extraCopy); + return command; + } + return null; + } + } + + public class HelpCommand : Command + { + public HelpCommand () + : base ("help", help: "Show this message and exit") + { + } + + public override int Invoke (IEnumerable arguments) + { + var extra = new List (arguments ?? new string [0]); + var _ = CommandSet.Options.MessageLocalizer; + if (extra.Count == 0) { + CommandSet.Options.WriteOptionDescriptions (CommandSet.Out); + return 0; + } + var command = CommandSet.GetCommand (extra); + if (command == this || extra.Contains ("--help")) { + CommandSet.Out.WriteLine (_ ($"Usage: {CommandSet.Suite} COMMAND [OPTIONS]")); + CommandSet.Out.WriteLine (_ ($"Use `{CommandSet.Suite} help COMMAND` for help on a specific command.")); + CommandSet.Out.WriteLine (); + CommandSet.Out.WriteLine (_ ($"Available commands:")); + CommandSet.Out.WriteLine (); + var commands = GetCommands (); + commands.Sort ((x, y) => string.Compare (x.Key, y.Key, StringComparison.OrdinalIgnoreCase)); + foreach (var c in commands) { + if (c.Key == "help") { + continue; + } + CommandSet.Options.WriteCommandDescription (CommandSet.Out, c.Value, c.Key); + } + CommandSet.Options.WriteCommandDescription (CommandSet.Out, CommandSet.help, "help"); + return 0; + } + if (command == null) { + WriteUnknownCommand (extra [0]); + return 1; + } + if (command.Options != null) { + command.Options.WriteOptionDescriptions (CommandSet.Out); + return 0; + } + return command.Invoke (new [] { "--help" }); + } + + List> GetCommands () + { + var commands = new List> (); + + foreach (var c in CommandSet) { + commands.Add (new KeyValuePair(c.Name, c)); + } + + if (CommandSet.NestedCommandSets == null) + return commands; + + foreach (var nc in CommandSet.NestedCommandSets) { + AddNestedCommands (commands, "", nc); + } + + return commands; + } + + void AddNestedCommands (List> commands, string outer, CommandSet value) + { + foreach (var v in value) { + commands.Add (new KeyValuePair($"{outer}{value.Suite} {v.Name}", v)); + } + if (value.NestedCommandSets == null) + return; + foreach (var nc in value.NestedCommandSets) { + AddNestedCommands (commands, $"{outer}{value.Suite} ", nc); + } + } + + internal void WriteUnknownCommand (string unknownCommand) + { + CommandSet.Error.WriteLine (CommandSet.Options.MessageLocalizer ($"{CommandSet.Suite}: Unknown command: {unknownCommand}")); + CommandSet.Error.WriteLine (CommandSet.Options.MessageLocalizer ($"{CommandSet.Suite}: Use `{CommandSet.Suite} help` for usage.")); + } + } +} + diff --git a/optimize-png-mbtiles/Program.cs b/optimize-png-mbtiles/Program.cs new file mode 100644 index 0000000..c398998 --- /dev/null +++ b/optimize-png-mbtiles/Program.cs @@ -0,0 +1,80 @@ +using System; +using System.IO; +using System.Collections.Generic; +using Mono.Options; + + +namespace optimizepngmbtiles { + + + public class TileData { + public int z; + public long x; + public long y; + } + + + public class Program { + + + static int Main(string[] args) { + + + CruncherSettings cs = new CruncherSettings(); + + var options = new OptionSet { + {"f|mbtiles=", $"Path to MBTiles", (string mb)=> cs.MbTilesPath=mb }, + {"m|min-quality=", $"PNG min quality [0..100]. Default:{cs.MinQuality}", (int mq)=> cs.MinQuality=mq }, + {"x|max-quality=", $"PNG max quality [0..100]. Default:{cs.MaxQuality}", (int mq)=> cs.MaxQuality=mq }, + {"s|speed=", $"PNG speed [1..10]. Slower better quality. Default:{cs.Speed}", (int s)=> cs.Speed=s }, + {"t|threads=", $"Threads. Default (Processors):{cs.Threads}", (int t)=> cs.Threads=t }, + {"b|batch-size=", $"Batch size. Number of tiles processed in one batch. Default:{cs.BatchSize}", (int bs)=>cs.BatchSize=bs } + }; + + options.WriteOptionDescriptions(Console.Out); + Console.WriteLine(); + + List extra = new List(); + try { + extra = options.Parse(args); + } + catch (OptionException oe) { + Console.WriteLine("Error parsing parameters:"); + Console.WriteLine(oe.Message); + return 1; + } + + if (extra.Count > 0) { + Console.WriteLine("Unknown parameters:"); + Console.WriteLine(string.Join(Environment.NewLine, extra)); + return 1; + } + + if (string.IsNullOrWhiteSpace(cs.MbTilesPath)) { + Console.WriteLine("Missing parameter: MBTiles file not specified"); + return 1; + } + + if (!File.Exists(cs.MbTilesPath)) { + Console.WriteLine($"MBTiles file does not exist: {cs.MbTilesPath}"); + return 1; + } + + + Console.WriteLine("Using parameters:"); + Console.WriteLine(cs.ToString()); + Console.WriteLine(); + + Cruncher cruncher = new Cruncher(cs); + cruncher.Go(); + + + return 0; + } + + + + + + } +} diff --git a/optimize-png-mbtiles/imagequant_sys.dll b/optimize-png-mbtiles/imagequant_sys.dll new file mode 100644 index 0000000..4d99808 Binary files /dev/null and b/optimize-png-mbtiles/imagequant_sys.dll differ diff --git a/optimize-png-mbtiles/optimizepngmbtiles.csproj b/optimize-png-mbtiles/optimizepngmbtiles.csproj new file mode 100644 index 0000000..bf16f47 --- /dev/null +++ b/optimize-png-mbtiles/optimizepngmbtiles.csproj @@ -0,0 +1,32 @@ + + + + true + Exe + netcoreapp3.1 + x64 + 0.0.0.1 + 0.0.0.1 + 0.0.0.1 + + + + + + + + + + Always + + + Always + + + + + + + + + diff --git a/optimize-png-mbtiles/optimizepngmbtiles.sln b/optimize-png-mbtiles/optimizepngmbtiles.sln new file mode 100644 index 0000000..7fb71ec --- /dev/null +++ b/optimize-png-mbtiles/optimizepngmbtiles.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29006.145 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "optimizepngmbtiles", "optimizepngmbtiles.csproj", "{9A2C4425-781E-4F3D-A044-CD60DC2EBE47}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {9A2C4425-781E-4F3D-A044-CD60DC2EBE47}.Debug|x64.ActiveCfg = Debug|x64 + {9A2C4425-781E-4F3D-A044-CD60DC2EBE47}.Debug|x64.Build.0 = Debug|x64 + {9A2C4425-781E-4F3D-A044-CD60DC2EBE47}.Release|x64.ActiveCfg = Release|x64 + {9A2C4425-781E-4F3D-A044-CD60DC2EBE47}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {EC57270F-ED60-4C83-AF72-503885F60A96} + EndGlobalSection +EndGlobal diff --git a/optimize-png-mbtiles/pngquant.exe b/optimize-png-mbtiles/pngquant.exe new file mode 100644 index 0000000..572318d Binary files /dev/null and b/optimize-png-mbtiles/pngquant.exe differ