diff --git a/benchmark/blobs_benchmark_results.json b/benchmark/blobs_benchmark_results.json new file mode 100644 index 00000000..14a834f8 --- /dev/null +++ b/benchmark/blobs_benchmark_results.json @@ -0,0 +1,262 @@ +{ + "25": { + "KernelHerding": { + "Unweighted_MMD": 0.024272788688540457, + "Unweighted_KSD": 0.07254663035273552, + "Weighted_MMD": 0.008470626920461655, + "Weighted_KSD": 0.07226677536964417, + "Time": 4.6005674901998646 + }, + "RandomSample": { + "Unweighted_MMD": 0.11142438650131226, + "Unweighted_KSD": 0.07730772346258163, + "Weighted_MMD": 0.011223642434924842, + "Weighted_KSD": 0.07383343055844308, + "Time": 3.495483254200008 + }, + "RPCholesky": { + "Unweighted_MMD": 0.14004717767238617, + "Unweighted_KSD": 0.059306026250123975, + "Weighted_MMD": 0.003688266733661294, + "Weighted_KSD": 0.07196912616491317, + "Time": 4.230013548799798 + }, + "SteinThinning": { + "Unweighted_MMD": 0.14796174466609954, + "Unweighted_KSD": 0.07581270486116409, + "Weighted_MMD": 0.01757101807743311, + "Weighted_KSD": 0.07423881441354752, + "Time": 4.806701995600088 + }, + "KernelThinning": { + "Unweighted_MMD": 0.014880230650305748, + "Unweighted_KSD": 0.07227124646306038, + "Weighted_MMD": 0.005388019885867834, + "Weighted_KSD": 0.07246261909604072, + "Time": 27.173368372400184 + }, + "CompressPlusPlus": { + "Unweighted_MMD": 0.013212332502007484, + "Unweighted_KSD": 0.07247907817363738, + "Weighted_MMD": 0.007080519571900368, + "Weighted_KSD": 0.07277720794081688, + "Time": 17.304506266200043 + }, + "ProbabilisticIterativeHerding": { + "Unweighted_MMD": 0.021128473989665508, + "Unweighted_KSD": 0.0732197754085064, + "Weighted_MMD": 0.007852014992386103, + "Weighted_KSD": 0.07306945249438286, + "Time": 4.6694933262000635 + }, + "IterativeHerding": { + "Unweighted_MMD": 0.007051250245422125, + "Unweighted_KSD": 0.07203583419322968, + "Weighted_MMD": 0.005125141562893986, + "Weighted_KSD": 0.07220595926046372, + "Time": 4.062583659599841 + }, + "CubicProbIterativeHerding": { + "Unweighted_MMD": 0.004542805999517441, + "Unweighted_KSD": 0.07216479405760765, + "Weighted_MMD": 0.003512424463406205, + "Weighted_KSD": 0.07236581966280937, + "Time": 4.687457689599796 + } + }, + "50": { + "KernelHerding": { + "Unweighted_MMD": 0.014010918885469436, + "Unweighted_KSD": 0.0722734160721302, + "Weighted_MMD": 0.0031911543337628245, + "Weighted_KSD": 0.07209383249282837, + "Time": 4.1393956179999805 + }, + "RandomSample": { + "Unweighted_MMD": 0.10492457151412964, + "Unweighted_KSD": 0.07875456660985947, + "Weighted_MMD": 0.004955455008894205, + "Weighted_KSD": 0.07259993627667427, + "Time": 3.580713712999932 + }, + "RPCholesky": { + "Unweighted_MMD": 0.1466503471136093, + "Unweighted_KSD": 0.056694062799215315, + "Weighted_MMD": 0.0015391094610095024, + "Weighted_KSD": 0.0722087174654007, + "Time": 3.8200428860001923 + }, + "SteinThinning": { + "Unweighted_MMD": 0.13258629888296128, + "Unweighted_KSD": 0.07708697170019149, + "Weighted_MMD": 0.006761046499013901, + "Weighted_KSD": 0.07263452410697938, + "Time": 4.231214966799962 + }, + "KernelThinning": { + "Unweighted_MMD": 0.006303768884390592, + "Unweighted_KSD": 0.07201230749487877, + "Weighted_MMD": 0.0022462865337729452, + "Weighted_KSD": 0.07222185432910919, + "Time": 15.216021602399996 + }, + "CompressPlusPlus": { + "Unweighted_MMD": 0.007616249471902847, + "Unweighted_KSD": 0.07215439230203628, + "Weighted_MMD": 0.0028188966680318117, + "Weighted_KSD": 0.07224903926253319, + "Time": 11.209934081999744 + }, + "ProbabilisticIterativeHerding": { + "Unweighted_MMD": 0.015107517503201962, + "Unweighted_KSD": 0.07347788587212563, + "Weighted_MMD": 0.003151226742193103, + "Weighted_KSD": 0.07250117510557175, + "Time": 4.343779678600185 + }, + "IterativeHerding": { + "Unweighted_MMD": 0.003708381252363324, + "Unweighted_KSD": 0.07212337255477905, + "Weighted_MMD": 0.002603885461576283, + "Weighted_KSD": 0.07219909951090812, + "Time": 3.6810207548000107 + }, + "CubicProbIterativeHerding": { + "Unweighted_MMD": 0.001733466051518917, + "Unweighted_KSD": 0.07222620248794556, + "Weighted_MMD": 0.001442490390036255, + "Weighted_KSD": 0.07229570895433426, + "Time": 4.199541498000144 + } + }, + "100": { + "KernelHerding": { + "Unweighted_MMD": 0.007909100409597159, + "Unweighted_KSD": 0.07176313027739525, + "Weighted_MMD": 0.0018589411629363894, + "Weighted_KSD": 0.07220481112599372, + "Time": 4.31388007539972 + }, + "RandomSample": { + "Unweighted_MMD": 0.05501915663480759, + "Unweighted_KSD": 0.07520547062158585, + "Weighted_MMD": 0.00180354667827487, + "Weighted_KSD": 0.07226956561207772, + "Time": 3.731109356599518 + }, + "RPCholesky": { + "Unweighted_MMD": 0.09764691218733787, + "Unweighted_KSD": 0.062210434675216676, + "Weighted_MMD": 0.0010440661339089275, + "Weighted_KSD": 0.07225104942917823, + "Time": 4.349850091400003 + }, + "SteinThinning": { + "Unweighted_MMD": 0.13784433156251907, + "Unweighted_KSD": 0.08129674047231675, + "Weighted_MMD": 0.0046910161152482035, + "Weighted_KSD": 0.07230838015675545, + "Time": 4.689982681799847 + }, + "KernelThinning": { + "Unweighted_MMD": 0.002685086103156209, + "Unweighted_KSD": 0.07206880524754525, + "Weighted_MMD": 0.001265210215933621, + "Weighted_KSD": 0.07226345017552376, + "Time": 10.10230621419978 + }, + "CompressPlusPlus": { + "Unweighted_MMD": 0.0029356910847127436, + "Unweighted_KSD": 0.07219576761126519, + "Weighted_MMD": 0.0012260458199307323, + "Weighted_KSD": 0.07228517681360244, + "Time": 9.244769073800308 + }, + "ProbabilisticIterativeHerding": { + "Unweighted_MMD": 0.009710153844207526, + "Unweighted_KSD": 0.07278616279363632, + "Weighted_MMD": 0.0018384325550869108, + "Weighted_KSD": 0.07236671000719071, + "Time": 4.425218307400064 + }, + "IterativeHerding": { + "Unweighted_MMD": 0.0022563493344932794, + "Unweighted_KSD": 0.07212945297360421, + "Weighted_MMD": 0.001406662119552493, + "Weighted_KSD": 0.07225525602698327, + "Time": 4.298705347399846 + }, + "CubicProbIterativeHerding": { + "Unweighted_MMD": 0.0008045180700719356, + "Unweighted_KSD": 0.07221448868513107, + "Weighted_MMD": 0.0009792268159799279, + "Weighted_KSD": 0.07225939556956291, + "Time": 4.68569216800006 + } + }, + "200": { + "KernelHerding": { + "Unweighted_MMD": 0.004258563183248043, + "Unweighted_KSD": 0.0720168687403202, + "Weighted_MMD": 0.0011734690284356474, + "Weighted_KSD": 0.0722421571612358, + "Time": 4.809446495800148 + }, + "RandomSample": { + "Unweighted_MMD": 0.04152125939726829, + "Unweighted_KSD": 0.07231617346405983, + "Weighted_MMD": 0.000913540180772543, + "Weighted_KSD": 0.0722603291273117, + "Time": 3.7448029847997533 + }, + "RPCholesky": { + "Unweighted_MMD": 0.05692300647497177, + "Unweighted_KSD": 0.0671866662800312, + "Weighted_MMD": 0.0008295111590996384, + "Weighted_KSD": 0.07224812433123588, + "Time": 4.360847868199926 + }, + "SteinThinning": { + "Unweighted_MMD": 0.14454428851604462, + "Unweighted_KSD": 0.08556406646966934, + "Weighted_MMD": 0.0028360273223370313, + "Weighted_KSD": 0.07215539738535881, + "Time": 4.83350045979987 + }, + "KernelThinning": { + "Unweighted_MMD": 0.0015182187082245946, + "Unweighted_KSD": 0.07213710397481918, + "Weighted_MMD": 0.000885988853406161, + "Weighted_KSD": 0.07226478308439255, + "Time": 6.940934421800193 + }, + "CompressPlusPlus": { + "Unweighted_MMD": 0.0014102120650932193, + "Unweighted_KSD": 0.07215408384799957, + "Weighted_MMD": 0.0007552313501946629, + "Weighted_KSD": 0.07224038168787957, + "Time": 7.29123429639967 + }, + "ProbabilisticIterativeHerding": { + "Unweighted_MMD": 0.006357756908982992, + "Unweighted_KSD": 0.07269964888691902, + "Weighted_MMD": 0.0008730732253752649, + "Weighted_KSD": 0.07227222323417663, + "Time": 4.814415276399814 + }, + "IterativeHerding": { + "Unweighted_MMD": 0.0013821582775563, + "Unweighted_KSD": 0.07216034978628158, + "Weighted_MMD": 0.0009947988553903997, + "Weighted_KSD": 0.07224116325378419, + "Time": 4.238990427600038 + }, + "CubicProbIterativeHerding": { + "Unweighted_MMD": 0.0005821106024086475, + "Unweighted_KSD": 0.07220486029982567, + "Weighted_MMD": 0.0007064452278427779, + "Weighted_KSD": 0.07225989773869515, + "Time": 4.936300538400064 + } + } +} \ No newline at end of file diff --git a/benchmark/david_benchmark_results.png b/benchmark/david_benchmark_results.png new file mode 100644 index 00000000..2046eb2b Binary files /dev/null and b/benchmark/david_benchmark_results.png differ diff --git a/benchmark/mnist_benchmark_results.json b/benchmark/mnist_benchmark_results.json index 4f357074..986b7848 100644 --- a/benchmark/mnist_benchmark_results.json +++ b/benchmark/mnist_benchmark_results.json @@ -2,939 +2,1207 @@ "Random Sample": { "25": { "0": { - "accuracy": 0.47499004006385803, - "time_taken": 23.352234674006468 + "accuracy": 0.49939975142478943, + "time_taken": 31.378794076000304 }, "1": { - "accuracy": 0.5094035863876343, - "time_taken": 24.481476221000776 + "accuracy": 0.4205680191516876, + "time_taken": 15.958629951000148 }, "2": { - "accuracy": 0.4429771900177002, - "time_taken": 25.436731434005196 + "accuracy": 0.4317728579044342, + "time_taken": 16.24605190999955 }, "3": { - "accuracy": 0.5266107320785522, - "time_taken": 21.81958907599619 + "accuracy": 0.4267708659172058, + "time_taken": 16.77121249100037 }, "4": { - "accuracy": 0.4708881676197052, - "time_taken": 25.530452502993285 + "accuracy": 0.41426557302474976, + "time_taken": 20.9866746280004 } }, "50": { "0": { - "accuracy": 0.6092996001243591, - "time_taken": 8.290402896993328 + "accuracy": 0.5915995836257935, + "time_taken": 13.46058772699962 }, "1": { - "accuracy": 0.6098998785018921, - "time_taken": 10.664244551997399 + "accuracy": 0.5806996822357178, + "time_taken": 7.307854431999658 }, "2": { - "accuracy": 0.5316997766494751, - "time_taken": 7.351946285998565 + "accuracy": 0.5749998092651367, + "time_taken": 14.328193377000389 }, "3": { - "accuracy": 0.6038997769355774, - "time_taken": 7.368035252991831 + "accuracy": 0.6124996542930603, + "time_taken": 10.097868826000195 }, "4": { - "accuracy": 0.6276999711990356, - "time_taken": 15.862566682990291 + "accuracy": 0.5182000994682312, + "time_taken": 13.699923365000359 } }, "100": { "0": { - "accuracy": 0.704299807548523, - "time_taken": 6.759049768996192 + "accuracy": 0.6564000844955444, + "time_taken": 6.6002176759993745 }, "1": { - "accuracy": 0.7321001887321472, - "time_taken": 5.9133299559907755 + "accuracy": 0.7256999611854553, + "time_taken": 4.555681074999484 }, "2": { - "accuracy": 0.7243001461029053, - "time_taken": 9.780433213003562 + "accuracy": 0.7220999598503113, + "time_taken": 5.915041250000286 }, "3": { - "accuracy": 0.7279003262519836, - "time_taken": 3.8765327779983636 + "accuracy": 0.687000036239624, + "time_taken": 5.589812125000208 }, "4": { - "accuracy": 0.6853998899459839, - "time_taken": 7.18066198201268 + "accuracy": 0.7198001146316528, + "time_taken": 9.042554675000247 } }, "500": { "0": { - "accuracy": 0.849459171295166, - "time_taken": 4.7403871529968455 + "accuracy": 0.8401442170143127, + "time_taken": 4.9041379580003195 }, "1": { - "accuracy": 0.8390424847602844, - "time_taken": 2.9221081489959033 + "accuracy": 0.8366386294364929, + "time_taken": 3.9820582559996183 }, "2": { - "accuracy": 0.8586738705635071, - "time_taken": 4.059734050009865 + "accuracy": 0.8532652258872986, + "time_taken": 4.150694207000015 }, "3": { - "accuracy": 0.8433493971824646, - "time_taken": 3.1853989910014207 + "accuracy": 0.8519631624221802, + "time_taken": 3.2897759840006984 }, "4": { - "accuracy": 0.8508613705635071, - "time_taken": 4.598661220996291 + "accuracy": 0.8568710088729858, + "time_taken": 3.2805830689994764 } }, "1000": { "0": { - "accuracy": 0.8806089758872986, - "time_taken": 4.128512479001074 + "accuracy": 0.8800080418586731, + "time_taken": 4.472438157999932 }, "1": { - "accuracy": 0.8755007982254028, - "time_taken": 2.8908094719954534 + "accuracy": 0.8753004670143127, + "time_taken": 3.1494866820003153 }, "2": { - "accuracy": 0.8828125, - "time_taken": 2.8451506499986863 + "accuracy": 0.8787059187889099, + "time_taken": 3.3807121429999825 }, "3": { - "accuracy": 0.8731971383094788, - "time_taken": 2.985724409998511 + "accuracy": 0.8870192170143127, + "time_taken": 3.851220937999642 }, "4": { - "accuracy": 0.8818109035491943, - "time_taken": 3.223807489994215 + "accuracy": 0.8785056471824646, + "time_taken": 3.1983475079996424 } }, "5000": { "0": { - "accuracy": 0.9258814454078674, - "time_taken": 4.811194872003398 + "accuracy": 0.9187700152397156, + "time_taken": 4.877470247000019 }, "1": { - "accuracy": 0.9238781929016113, - "time_taken": 3.5581146879994776 + "accuracy": 0.9220753312110901, + "time_taken": 4.563210882000021 }, "2": { - "accuracy": 0.9277844429016113, - "time_taken": 3.5328966109955218 + "accuracy": 0.9268830418586731, + "time_taken": 4.27181648900023 }, "3": { - "accuracy": 0.9291867017745972, - "time_taken": 5.172802444998524 + "accuracy": 0.9260817170143127, + "time_taken": 4.163932319000196 }, "4": { - "accuracy": 0.9294871687889099, - "time_taken": 3.727755020998302 + "accuracy": 0.9253806471824646, + "time_taken": 5.401279224999598 } } }, "RP Cholesky": { "25": { "0": { - "accuracy": 0.48649486899375916, - "time_taken": 15.06727149800281 + "accuracy": 0.5479191541671753, + "time_taken": 30.12485848600045 }, "1": { - "accuracy": 0.5369148850440979, - "time_taken": 16.84173994199955 + "accuracy": 0.5831333994865417, + "time_taken": 17.1290923939996 }, "2": { - "accuracy": 0.5941379070281982, - "time_taken": 28.234114010003395 + "accuracy": 0.506402313709259, + "time_taken": 28.668303671999638 }, "3": { - "accuracy": 0.5332134366035461, - "time_taken": 22.014903976989444 + "accuracy": 0.48569461703300476, + "time_taken": 22.65919453600054 }, "4": { - "accuracy": 0.5119049549102783, - "time_taken": 24.88884191699617 + "accuracy": 0.4908963739871979, + "time_taken": 25.84216402400034 } }, "50": { "0": { - "accuracy": 0.5625001192092896, - "time_taken": 15.471359351999126 + "accuracy": 0.5573997497558594, + "time_taken": 11.868133194999245 }, "1": { - "accuracy": 0.6616997122764587, - "time_taken": 9.69486076499743 + "accuracy": 0.6715995669364929, + "time_taken": 9.038644168000246 }, "2": { - "accuracy": 0.5726996660232544, - "time_taken": 7.714200211994466 + "accuracy": 0.5750996470451355, + "time_taken": 15.650777019999623 }, "3": { - "accuracy": 0.633499801158905, - "time_taken": 10.867528442002367 + "accuracy": 0.615599513053894, + "time_taken": 17.46370884499993 }, "4": { - "accuracy": 0.6135995984077454, - "time_taken": 17.245890217003762 + "accuracy": 0.6460996270179749, + "time_taken": 12.382691423999859 } }, "100": { "0": { - "accuracy": 0.6975999474525452, - "time_taken": 13.063453700000537 + "accuracy": 0.7053002119064331, + "time_taken": 13.038988242000414 }, "1": { - "accuracy": 0.7179999947547913, - "time_taken": 7.070716499001719 + "accuracy": 0.6796000599861145, + "time_taken": 5.734187043999555 }, "2": { - "accuracy": 0.6369000673294067, - "time_taken": 5.612793920998229 + "accuracy": 0.6684001684188843, + "time_taken": 10.304006375000426 }, "3": { - "accuracy": 0.6892000436782837, - "time_taken": 6.446513513001264 + "accuracy": 0.7305001020431519, + "time_taken": 6.787468426000487 }, "4": { - "accuracy": 0.6939999461174011, - "time_taken": 8.73156723100692 + "accuracy": 0.6646999716758728, + "time_taken": 8.555311609000455 } }, "500": { "0": { - "accuracy": 0.8505609035491943, - "time_taken": 6.197642628001631 + "accuracy": 0.8393429517745972, + "time_taken": 5.700323695000407 }, "1": { - "accuracy": 0.8322315812110901, - "time_taken": 3.782923115999438 + "accuracy": 0.8234174847602844, + "time_taken": 4.577245696000318 }, "2": { - "accuracy": 0.8162059187889099, - "time_taken": 4.230790430010529 + "accuracy": 0.8246194124221802, + "time_taken": 4.333464565000213 }, "3": { - "accuracy": 0.8254206776618958, - "time_taken": 4.489102895997348 + "accuracy": 0.8459535241127014, + "time_taken": 4.002924071000052 }, "4": { - "accuracy": 0.8451522588729858, - "time_taken": 3.9125116750074085 + "accuracy": 0.8322315812110901, + "time_taken": 4.44240554299995 } }, "1000": { "0": { - "accuracy": 0.8621795177459717, - "time_taken": 6.000493390994961 + "accuracy": 0.8704928159713745, + "time_taken": 6.094056519999867 }, "1": { - "accuracy": 0.8517628312110901, - "time_taken": 4.43663791801373 + "accuracy": 0.8677884936332703, + "time_taken": 4.771197477999522 }, "2": { - "accuracy": 0.8653846383094788, - "time_taken": 4.807365334010683 + "accuracy": 0.8700921535491943, + "time_taken": 5.191751323000062 }, "3": { - "accuracy": 0.8575721383094788, - "time_taken": 4.603306793986121 + "accuracy": 0.8765023946762085, + "time_taken": 5.251704577000055 }, "4": { - "accuracy": 0.8751001954078674, - "time_taken": 4.6778737399872625 + "accuracy": 0.857271671295166, + "time_taken": 5.254659973999878 } }, "5000": { "0": { - "accuracy": 0.9258814454078674, - "time_taken": 30.59640652500093 + "accuracy": 0.9259815812110901, + "time_taken": 31.580425969000316 }, "1": { - "accuracy": 0.9270833730697632, - "time_taken": 28.73688672199205 + "accuracy": 0.9291867017745972, + "time_taken": 30.337975878999714 }, "2": { - "accuracy": 0.9242788553237915, - "time_taken": 27.798969063995173 + "accuracy": 0.9300881624221802, + "time_taken": 31.068755215000238 }, "3": { - "accuracy": 0.9295873641967773, - "time_taken": 28.526359000999946 + "accuracy": 0.9288862347602844, + "time_taken": 28.69117661600012 }, "4": { - "accuracy": 0.9217748641967773, - "time_taken": 28.89112439600285 + "accuracy": 0.9255809187889099, + "time_taken": 28.678187123000498 } } }, "Kernel Herding": { "25": { "0": { - "accuracy": 0.41976743936538696, - "time_taken": 16.602575227007037 + "accuracy": 0.36344558000564575, + "time_taken": 27.81852218200038 }, "1": { - "accuracy": 0.3865547478199005, - "time_taken": 39.596862955004326 + "accuracy": 0.40876394510269165, + "time_taken": 28.917832838000322 }, "2": { - "accuracy": 0.4191674292087555, - "time_taken": 27.549024899999495 + "accuracy": 0.37885135412216187, + "time_taken": 33.120471524000095 }, "3": { - "accuracy": 0.3826534152030945, - "time_taken": 15.041240438004024 + "accuracy": 0.3827531635761261, + "time_taken": 16.74550173100033 }, "4": { - "accuracy": 0.36784714460372925, - "time_taken": 31.221653377986513 + "accuracy": 0.3984594941139221, + "time_taken": 31.39536281100027 } }, "50": { "0": { - "accuracy": 0.4702000617980957, - "time_taken": 9.662910286991973 + "accuracy": 0.5153999328613281, + "time_taken": 8.394265891000032 }, "1": { - "accuracy": 0.49190017580986023, - "time_taken": 8.503036755995709 + "accuracy": 0.5643997192382812, + "time_taken": 8.034130296000512 }, "2": { - "accuracy": 0.4674000144004822, - "time_taken": 6.310842988008517 + "accuracy": 0.5117997527122498, + "time_taken": 7.021193917999881 }, "3": { - "accuracy": 0.46950021386146545, - "time_taken": 5.354004640001222 + "accuracy": 0.5069000720977783, + "time_taken": 8.373046021999471 }, "4": { - "accuracy": 0.5017001032829285, - "time_taken": 8.116046239010757 + "accuracy": 0.5413997173309326, + "time_taken": 8.041774192999583 } }, "100": { "0": { - "accuracy": 0.6386004090309143, - "time_taken": 7.6281732080096845 + "accuracy": 0.6686000823974609, + "time_taken": 6.880627470000036 }, "1": { - "accuracy": 0.6136000752449036, - "time_taken": 5.189364300007583 + "accuracy": 0.6687001585960388, + "time_taken": 5.998970880000343 }, "2": { - "accuracy": 0.6494002342224121, - "time_taken": 5.8485669609945035 + "accuracy": 0.6764000654220581, + "time_taken": 6.708478034000109 }, "3": { - "accuracy": 0.639400064945221, - "time_taken": 5.13314476099913 + "accuracy": 0.6682003140449524, + "time_taken": 7.342494532999808 }, "4": { - "accuracy": 0.626500129699707, - "time_taken": 5.724788473991794 + "accuracy": 0.672999918460846, + "time_taken": 6.854923852999491 } }, "500": { "0": { - "accuracy": 0.796875, - "time_taken": 5.436996539996471 + "accuracy": 0.7989783883094788, + "time_taken": 5.009305618999861 }, "1": { - "accuracy": 0.7882612347602844, - "time_taken": 3.7532910439913394 + "accuracy": 0.7846554517745972, + "time_taken": 3.426840018999428 }, "2": { - "accuracy": 0.7895632982254028, - "time_taken": 3.608708291008952 + "accuracy": 0.7994791865348816, + "time_taken": 3.6989639409994197 }, "3": { - "accuracy": 0.8052884936332703, - "time_taken": 3.843011178993038 + "accuracy": 0.7809495329856873, + "time_taken": 3.7089166180003303 }, "4": { - "accuracy": 0.7890625, - "time_taken": 4.171767199004535 + "accuracy": 0.7839543223381042, + "time_taken": 3.630458699999508 } }, "1000": { "0": { - "accuracy": 0.8404447436332703, - "time_taken": 4.949075399010326 + "accuracy": 0.8508613705635071, + "time_taken": 5.865284939999583 }, "1": { - "accuracy": 0.8525640964508057, - "time_taken": 3.195582884000032 + "accuracy": 0.8452523946762085, + "time_taken": 3.5790511420000257 }, "2": { - "accuracy": 0.848557710647583, - "time_taken": 3.2079661229945486 + "accuracy": 0.8441506624221802, + "time_taken": 3.766006290000405 }, "3": { - "accuracy": 0.8504607677459717, - "time_taken": 3.3226542650081683 + "accuracy": 0.8497596383094788, + "time_taken": 3.753327657000227 }, "4": { - "accuracy": 0.8540664911270142, - "time_taken": 3.8391184149950277 + "accuracy": 0.8400440812110901, + "time_taken": 3.5520159130001048 } }, "5000": { "0": { - "accuracy": 0.9304887652397156, - "time_taken": 7.827742292007315 + "accuracy": 0.9318910241127014, + "time_taken": 8.529647805000423 }, "1": { - "accuracy": 0.9318910241127014, - "time_taken": 5.016722931992263 + "accuracy": 0.9282852411270142, + "time_taken": 5.212250545999268 }, "2": { - "accuracy": 0.9341947436332703, - "time_taken": 5.358994267997332 + "accuracy": 0.9311898946762085, + "time_taken": 6.773341258000073 }, "3": { - "accuracy": 0.9357972741127014, - "time_taken": 6.045616682997206 + "accuracy": 0.9261819124221802, + "time_taken": 5.1819647869997425 }, "4": { - "accuracy": 0.9310897588729858, - "time_taken": 5.193479452995234 + "accuracy": 0.9283854365348816, + "time_taken": 6.26639689700005 } } }, "Stein Thinning": { "25": { "0": { - "accuracy": 0.3557422161102295, - "time_taken": 14.798573687992757 + "accuracy": 0.36664658784866333, + "time_taken": 26.155145622999953 }, "1": { - "accuracy": 0.37554997205734253, - "time_taken": 19.849432048999006 + "accuracy": 0.35044023394584656, + "time_taken": 16.872243384000285 }, "2": { - "accuracy": 0.365746408700943, - "time_taken": 15.417942133004544 + "accuracy": 0.33363354206085205, + "time_taken": 16.71151909799937 }, "3": { - "accuracy": 0.34683868288993835, - "time_taken": 14.829650210987893 + "accuracy": 0.3225291073322296, + "time_taken": 18.860941359000208 }, "4": { - "accuracy": 0.3714485466480255, - "time_taken": 16.72217624800396 + "accuracy": 0.342837393283844, + "time_taken": 15.351801773999796 } }, "50": { "0": { - "accuracy": 0.44350001215934753, - "time_taken": 11.345535130007192 + "accuracy": 0.3817000985145569, + "time_taken": 13.093195344000378 }, "1": { - "accuracy": 0.4461999833583832, - "time_taken": 13.185699121997459 + "accuracy": 0.3882002532482147, + "time_taken": 11.234985342999607 }, "2": { - "accuracy": 0.45210036635398865, - "time_taken": 10.52003458699619 + "accuracy": 0.4043000638484955, + "time_taken": 11.797061467000276 }, "3": { - "accuracy": 0.4285001754760742, - "time_taken": 11.961328809004044 + "accuracy": 0.3527999520301819, + "time_taken": 11.524091536000014 }, "4": { - "accuracy": 0.42170006036758423, - "time_taken": 10.88203769500251 + "accuracy": 0.38370028138160706, + "time_taken": 11.592887677000363 } }, "100": { "0": { - "accuracy": 0.5332000851631165, - "time_taken": 9.059783407006762 + "accuracy": 0.47479987144470215, + "time_taken": 9.806702724999923 }, "1": { - "accuracy": 0.4996998608112335, - "time_taken": 8.194069575009053 + "accuracy": 0.4605000913143158, + "time_taken": 10.19817711199994 }, "2": { - "accuracy": 0.5352000594139099, - "time_taken": 8.481501229995047 + "accuracy": 0.5033999085426331, + "time_taken": 9.12059926999973 }, "3": { - "accuracy": 0.4972999095916748, - "time_taken": 8.426191655002185 + "accuracy": 0.4161998927593231, + "time_taken": 8.590373965999788 }, "4": { - "accuracy": 0.4962000250816345, - "time_taken": 8.92351122800028 + "accuracy": 0.4584999680519104, + "time_taken": 9.745934776000468 } }, "500": { "0": { - "accuracy": 0.6190905570983887, - "time_taken": 9.206097094007418 + "accuracy": 0.5769230723381042, + "time_taken": 8.572717877999821 }, "1": { - "accuracy": 0.6015625, - "time_taken": 9.124566219994449 + "accuracy": 0.549879789352417, + "time_taken": 8.685776891999922 }, "2": { - "accuracy": 0.6050680875778198, - "time_taken": 8.69083937999676 + "accuracy": 0.5654046535491943, + "time_taken": 8.738311274999432 }, "3": { - "accuracy": 0.5941506624221802, - "time_taken": 9.4010932680103 + "accuracy": 0.5692107677459717, + "time_taken": 8.89052372200058 }, "4": { - "accuracy": 0.5866386294364929, - "time_taken": 9.061567828000989 + "accuracy": 0.5557892918586731, + "time_taken": 8.744390723999459 } }, "1000": { "0": { - "accuracy": 0.611177921295166, - "time_taken": 9.522722403999069 + "accuracy": 0.5848357677459717, + "time_taken": 10.872244053999566 }, "1": { - "accuracy": 0.6095753312110901, - "time_taken": 9.849882042995887 + "accuracy": 0.579026460647583, + "time_taken": 10.691493848000391 }, "2": { - "accuracy": 0.6153846383094788, - "time_taken": 10.069659473010688 + "accuracy": 0.5827323794364929, + "time_taken": 10.416130721000627 }, "3": { - "accuracy": 0.6005609035491943, - "time_taken": 10.171524318007869 + "accuracy": 0.5981570482254028, + "time_taken": 10.580251977999978 }, "4": { - "accuracy": 0.6041666865348816, - "time_taken": 10.320493794002687 + "accuracy": 0.5770232677459717, + "time_taken": 9.95398137400025 } }, "5000": { "0": { - "accuracy": 0.6329126954078674, - "time_taken": 29.329851916991174 + "accuracy": 0.6365184187889099, + "time_taken": 31.84610059700026 }, "1": { - "accuracy": 0.6323117017745972, - "time_taken": 29.234513267001603 + "accuracy": 0.6400240659713745, + "time_taken": 31.580490523000663 }, "2": { - "accuracy": 0.6167868375778198, - "time_taken": 33.62631028499163 + "accuracy": 0.6353164911270142, + "time_taken": 30.96478199500052 }, "3": { - "accuracy": 0.6232972741127014, - "time_taken": 29.677587008001865 + "accuracy": 0.6447315812110901, + "time_taken": 32.943920733000596 }, "4": { - "accuracy": 0.6148838400840759, - "time_taken": 29.441241257009096 + "accuracy": 0.6456330418586731, + "time_taken": 30.633445284999652 } } }, "Kernel Thinning": { "25": { "0": { - "accuracy": 0.5086035132408142, - "time_taken": 86.50353899000038 + "accuracy": 0.5464185476303101, + "time_taken": 110.06621436300065 }, "1": { - "accuracy": 0.5096036195755005, - "time_taken": 27.584838315000525 + "accuracy": 0.46028411388397217, + "time_taken": 25.186751725999784 }, "2": { - "accuracy": 0.4775909185409546, - "time_taken": 25.87109093600884 + "accuracy": 0.43417370319366455, + "time_taken": 19.643712891000177 }, "3": { - "accuracy": 0.4638856053352356, - "time_taken": 18.325479343999177 + "accuracy": 0.3735492527484894, + "time_taken": 19.60971717100074 }, "4": { - "accuracy": 0.37675046920776367, - "time_taken": 14.236614166002255 + "accuracy": 0.4456782639026642, + "time_taken": 20.31466248600009 } }, "50": { "0": { - "accuracy": 0.630199670791626, - "time_taken": 46.260885070994846 + "accuracy": 0.6653998494148254, + "time_taken": 59.023018455000056 }, "1": { - "accuracy": 0.6344994902610779, - "time_taken": 10.804280167998513 + "accuracy": 0.6325995326042175, + "time_taken": 11.286527774000206 }, "2": { - "accuracy": 0.5950995683670044, - "time_taken": 12.697421856006258 + "accuracy": 0.5777995586395264, + "time_taken": 10.881928441000127 }, "3": { - "accuracy": 0.5381998419761658, - "time_taken": 13.086974981997628 + "accuracy": 0.5715998411178589, + "time_taken": 10.913744210000004 }, "4": { - "accuracy": 0.5856001377105713, - "time_taken": 7.364729672000976 + "accuracy": 0.6640992760658264, + "time_taken": 12.46789212099975 } }, "100": { "0": { - "accuracy": 0.7089999914169312, - "time_taken": 27.71084121799504 + "accuracy": 0.7268001437187195, + "time_taken": 35.0293219160003 }, "1": { - "accuracy": 0.7250999212265015, - "time_taken": 6.485319369006902 + "accuracy": 0.7283000946044922, + "time_taken": 5.59897917499984 }, "2": { - "accuracy": 0.735200047492981, - "time_taken": 6.369364660000429 + "accuracy": 0.7609001994132996, + "time_taken": 6.843098536999605 }, "3": { - "accuracy": 0.6910000443458557, - "time_taken": 4.998287808994064 + "accuracy": 0.7039998173713684, + "time_taken": 5.921434724000392 }, "4": { - "accuracy": 0.7174001336097717, - "time_taken": 5.89240234499448 + "accuracy": 0.6804999113082886, + "time_taken": 5.160561370000323 } }, "500": { "0": { - "accuracy": 0.8403445482254028, - "time_taken": 10.419982768013142 + "accuracy": 0.8551682829856873, + "time_taken": 12.34510114800014 }, "1": { - "accuracy": 0.8527644276618958, - "time_taken": 3.7628963680035667 + "accuracy": 0.8511618971824646, + "time_taken": 4.445879892999983 }, "2": { - "accuracy": 0.858473539352417, - "time_taken": 3.6505097879999084 + "accuracy": 0.8497596383094788, + "time_taken": 3.9442537409995566 }, "3": { - "accuracy": 0.8390424847602844, - "time_taken": 3.832862326002214 + "accuracy": 0.8512620329856873, + "time_taken": 4.5083901919997516 }, "4": { - "accuracy": 0.8510617017745972, - "time_taken": 3.9164835209958255 + "accuracy": 0.838942289352417, + "time_taken": 3.8187096999999994 } }, "1000": { "0": { - "accuracy": 0.8800080418586731, - "time_taken": 10.038269149008556 + "accuracy": 0.8794070482254028, + "time_taken": 11.011465002000477 }, "1": { - "accuracy": 0.8777043223381042, - "time_taken": 3.7039374739979394 + "accuracy": 0.875901460647583, + "time_taken": 3.8713226500003657 }, "2": { - "accuracy": 0.8801081776618958, - "time_taken": 3.2720982680039015 + "accuracy": 0.8819110989570618, + "time_taken": 8.961700078999456 }, "3": { - "accuracy": 0.879807710647583, - "time_taken": 3.7813302020076662 + "accuracy": 0.8772035241127014, + "time_taken": 3.7053063889998157 }, "4": { - "accuracy": 0.8847155570983887, - "time_taken": 3.6361338120041182 + "accuracy": 0.8855168223381042, + "time_taken": 3.8933140440003626 } }, "5000": { "0": { - "accuracy": 0.9284855723381042, - "time_taken": 16.11581551800191 + "accuracy": 0.9211738705635071, + "time_taken": 15.707877896000355 }, "1": { - "accuracy": 0.9262820482254028, - "time_taken": 5.4665557440021075 + "accuracy": 0.9233773946762085, + "time_taken": 6.091783504999512 }, "2": { - "accuracy": 0.9265825152397156, - "time_taken": 6.757973326995852 + "accuracy": 0.9256811141967773, + "time_taken": 5.776395839000543 }, "3": { - "accuracy": 0.9316906929016113, - "time_taken": 6.069867120997515 + "accuracy": 0.926682710647583, + "time_taken": 6.179523726000298 }, "4": { - "accuracy": 0.9258814454078674, - "time_taken": 5.646533480001381 + "accuracy": 0.926682710647583, + "time_taken": 5.389667761000055 } } }, "Compress++": { "25": { "0": { - "accuracy": 0.5006005764007568, - "time_taken": 48.42374569299864 + "accuracy": 0.48279327154159546, + "time_taken": 63.73896679699919 }, "1": { - "accuracy": 0.42466992139816284, - "time_taken": 15.439281265003956 + "accuracy": 0.49789944291114807, + "time_taken": 19.31496702000004 }, "2": { - "accuracy": 0.4636856019496918, - "time_taken": 27.897809760004748 + "accuracy": 0.46188488602638245, + "time_taken": 19.98056298300071 }, "3": { - "accuracy": 0.4906962215900421, - "time_taken": 13.080031949997647 + "accuracy": 0.46188458800315857, + "time_taken": 24.72102610499951 }, "4": { - "accuracy": 0.4217684864997864, - "time_taken": 20.284750368999084 + "accuracy": 0.4839934706687927, + "time_taken": 38.3823201969999 } }, "50": { "0": { - "accuracy": 0.5905998349189758, - "time_taken": 25.081396187990322 + "accuracy": 0.5958995819091797, + "time_taken": 30.356294965000416 }, "1": { - "accuracy": 0.608699381351471, - "time_taken": 6.843037225000444 + "accuracy": 0.6557992696762085, + "time_taken": 10.960626758999751 }, "2": { - "accuracy": 0.58219975233078, - "time_taken": 7.7891707769886125 + "accuracy": 0.6099992394447327, + "time_taken": 12.611458585000037 }, "3": { - "accuracy": 0.4834998846054077, - "time_taken": 7.2219549550063675 + "accuracy": 0.6473994255065918, + "time_taken": 10.239416935000008 }, "4": { - "accuracy": 0.5665996670722961, - "time_taken": 9.369507609997527 + "accuracy": 0.5785996317863464, + "time_taken": 9.679247836000286 } }, "100": { "0": { - "accuracy": 0.7213001251220703, - "time_taken": 17.787696682993555 + "accuracy": 0.7106999754905701, + "time_taken": 21.78798379 }, "1": { - "accuracy": 0.7342995405197144, - "time_taken": 3.8416210310097085 + "accuracy": 0.7305997014045715, + "time_taken": 5.884108313999604 }, "2": { - "accuracy": 0.6959000825881958, - "time_taken": 4.702277585005504 + "accuracy": 0.7083998918533325, + "time_taken": 5.77458919899982 }, "3": { - "accuracy": 0.7541998028755188, - "time_taken": 4.814263307998772 + "accuracy": 0.6870997548103333, + "time_taken": 5.419124719000138 }, "4": { - "accuracy": 0.7210000157356262, - "time_taken": 4.949811800004682 + "accuracy": 0.7167999148368835, + "time_taken": 5.903611237000405 } }, "500": { "0": { - "accuracy": 0.8381410241127014, - "time_taken": 7.494220376000158 + "accuracy": 0.8560697436332703, + "time_taken": 8.24205664999954 }, "1": { - "accuracy": 0.8441506624221802, - "time_taken": 3.3129628419992514 + "accuracy": 0.8601762652397156, + "time_taken": 3.8466831529995034 }, "2": { - "accuracy": 0.8579727411270142, - "time_taken": 3.4562966030061943 + "accuracy": 0.846754789352417, + "time_taken": 4.29086993799956 }, "3": { - "accuracy": 0.8535656929016113, - "time_taken": 3.3829493960074615 + "accuracy": 0.8469551205635071, + "time_taken": 3.9742898100003003 }, "4": { - "accuracy": 0.8521634936332703, - "time_taken": 3.694734361008159 + "accuracy": 0.8459535241127014, + "time_taken": 3.465456570999777 } }, "1000": { "0": { - "accuracy": 0.8771033883094788, - "time_taken": 7.094734907004749 + "accuracy": 0.8879206776618958, + "time_taken": 7.61852171999999 }, "1": { - "accuracy": 0.8756009936332703, - "time_taken": 3.0448469719995046 + "accuracy": 0.876802921295166, + "time_taken": 3.580353852999906 }, "2": { - "accuracy": 0.8830128312110901, - "time_taken": 3.1766963270056294 + "accuracy": 0.8792067170143127, + "time_taken": 4.019800891000159 }, "3": { - "accuracy": 0.8806089758872986, - "time_taken": 3.6595106110034976 + "accuracy": 0.8861178159713745, + "time_taken": 3.750217070999497 }, "4": { - "accuracy": 0.8797075152397156, - "time_taken": 3.1855167699977756 + "accuracy": 0.8814102411270142, + "time_taken": 3.537440410000272 } }, "5000": { "0": { - "accuracy": 0.9220753312110901, - "time_taken": 6.997968723997474 + "accuracy": 0.9228765964508057, + "time_taken": 6.762459595999644 }, "1": { - "accuracy": 0.9252804517745972, - "time_taken": 4.05370223299542 + "accuracy": 0.9269831776618958, + "time_taken": 5.589771165999991 }, "2": { - "accuracy": 0.9270833730697632, - "time_taken": 4.276571049995255 + "accuracy": 0.9271835088729858, + "time_taken": 5.489222301999689 }, "3": { - "accuracy": 0.9282852411270142, - "time_taken": 4.490389550992404 + "accuracy": 0.9243789911270142, + "time_taken": 5.061679284000093 }, "4": { - "accuracy": 0.9259815812110901, - "time_taken": 4.635322599002393 + "accuracy": 0.9262820482254028, + "time_taken": 5.058138197999142 } } }, - "Probabilistic Iterative Herding": { + "Iterative Probabilistic Herding (constant)": { "25": { "0": { - "accuracy": 0.5539213418960571, - "time_taken": 22.00237893400481 + "accuracy": 0.565226137638092, + "time_taken": 25.58306674800042 }, "1": { - "accuracy": 0.5617246031761169, - "time_taken": 30.902002311995602 + "accuracy": 0.5584231019020081, + "time_taken": 26.9211215360001 }, "2": { - "accuracy": 0.5159062147140503, - "time_taken": 13.908021073002601 + "accuracy": 0.5649256110191345, + "time_taken": 24.513512876999812 }, "3": { - "accuracy": 0.4784916043281555, - "time_taken": 30.6841775149951 + "accuracy": 0.5609245300292969, + "time_taken": 27.941308749999735 }, "4": { - "accuracy": 0.5513207316398621, - "time_taken": 23.426923535997048 + "accuracy": 0.5159065127372742, + "time_taken": 19.32287024500056 } }, "50": { "0": { - "accuracy": 0.6849998235702515, - "time_taken": 13.994797082996229 + "accuracy": 0.6795998811721802, + "time_taken": 12.993868425000073 }, "1": { - "accuracy": 0.6985998153686523, - "time_taken": 12.24240016200929 + "accuracy": 0.6618993282318115, + "time_taken": 13.741371570999945 }, "2": { - "accuracy": 0.6409992575645447, - "time_taken": 10.90571705000184 + "accuracy": 0.7035000324249268, + "time_taken": 12.33730979000029 }, "3": { - "accuracy": 0.6208995580673218, - "time_taken": 9.279130113995052 + "accuracy": 0.6759998798370361, + "time_taken": 10.023791417000211 }, "4": { - "accuracy": 0.6784992814064026, - "time_taken": 8.457788833999075 + "accuracy": 0.6457993388175964, + "time_taken": 11.025239813000553 } }, "100": { "0": { - "accuracy": 0.7381998300552368, - "time_taken": 11.28686753900547 + "accuracy": 0.7630999088287354, + "time_taken": 10.076966365999397 }, "1": { - "accuracy": 0.7427999973297119, - "time_taken": 5.407744587995694 + "accuracy": 0.7409002780914307, + "time_taken": 7.087535218999619 }, "2": { - "accuracy": 0.7714999914169312, - "time_taken": 7.464625088003231 + "accuracy": 0.7367997765541077, + "time_taken": 6.279074885999762 }, "3": { - "accuracy": 0.7434998154640198, - "time_taken": 6.5014143439912 + "accuracy": 0.7450000047683716, + "time_taken": 6.497801151000203 }, "4": { - "accuracy": 0.7387998104095459, - "time_taken": 6.165277772000991 + "accuracy": 0.7276996970176697, + "time_taken": 6.045073967999997 } }, "500": { "0": { - "accuracy": 0.8561698794364929, - "time_taken": 8.956057899005827 + "accuracy": 0.8546674847602844, + "time_taken": 7.87245468600031 }, "1": { - "accuracy": 0.858473539352417, - "time_taken": 4.108647701999871 + "accuracy": 0.8464543223381042, + "time_taken": 4.942239801000142 }, "2": { - "accuracy": 0.8372396230697632, - "time_taken": 4.213687284005573 + "accuracy": 0.8433493971824646, + "time_taken": 5.234136874999422 }, "3": { - "accuracy": 0.8531650900840759, - "time_taken": 5.0821179729973665 + "accuracy": 0.8550681471824646, + "time_taken": 4.858938104000117 }, "4": { - "accuracy": 0.849459171295166, - "time_taken": 4.7921027800039155 + "accuracy": 0.856370210647583, + "time_taken": 5.168977192999591 } }, "1000": { "0": { - "accuracy": 0.8831129670143127, - "time_taken": 9.668020572993555 + "accuracy": 0.8842147588729858, + "time_taken": 8.376013503999275 + }, + "1": { + "accuracy": 0.8757011294364929, + "time_taken": 5.392493352999736 + }, + "2": { + "accuracy": 0.8839142918586731, + "time_taken": 4.918340146000446 + }, + "3": { + "accuracy": 0.884615421295166, + "time_taken": 4.754172397999355 + }, + "4": { + "accuracy": 0.8897235989570618, + "time_taken": 5.224267777000023 + } + }, + "5000": { + "0": { + "accuracy": 0.9242788553237915, + "time_taken": 21.26693089299988 + }, + "1": { + "accuracy": 0.9253806471824646, + "time_taken": 14.351450629999817 + }, + "2": { + "accuracy": 0.9214743971824646, + "time_taken": 14.56304887999977 + }, + "3": { + "accuracy": 0.9211738705635071, + "time_taken": 15.201094977000139 + }, + "4": { + "accuracy": 0.9302884936332703, + "time_taken": 15.344466227999874 + } + } + }, + "Iterative Herding": { + "25": { + "0": { + "accuracy": 0.469487726688385, + "time_taken": 26.213904901000205 + }, + "1": { + "accuracy": 0.4717889130115509, + "time_taken": 40.966588446999594 + }, + "2": { + "accuracy": 0.4678872227668762, + "time_taken": 17.88325883499965 + }, + "3": { + "accuracy": 0.4661862552165985, + "time_taken": 28.512475307000386 + }, + "4": { + "accuracy": 0.4657862186431885, + "time_taken": 21.193546985000467 + } + }, + "50": { + "0": { + "accuracy": 0.6057994365692139, + "time_taken": 16.804117133000545 + }, + "1": { + "accuracy": 0.5874000191688538, + "time_taken": 10.829441335999945 + }, + "2": { + "accuracy": 0.5901995897293091, + "time_taken": 14.236112396999488 + }, + "3": { + "accuracy": 0.5842994451522827, + "time_taken": 10.729420103999473 + }, + "4": { + "accuracy": 0.593899667263031, + "time_taken": 18.04358775899982 + } + }, + "100": { + "0": { + "accuracy": 0.7108997106552124, + "time_taken": 6.8043059060000814 + }, + "1": { + "accuracy": 0.7032998204231262, + "time_taken": 5.164458477000153 + }, + "2": { + "accuracy": 0.7057000398635864, + "time_taken": 5.732392590000018 + }, + "3": { + "accuracy": 0.689500093460083, + "time_taken": 5.783705160999489 + }, + "4": { + "accuracy": 0.676800012588501, + "time_taken": 5.26132266500008 + } + }, + "500": { + "0": { + "accuracy": 0.8225160241127014, + "time_taken": 6.017370114999721 }, "1": { - "accuracy": 0.8720953464508057, - "time_taken": 4.814860455997405 + "accuracy": 0.8190104365348816, + "time_taken": 4.340642064999884 }, "2": { - "accuracy": 0.8809094429016113, - "time_taken": 4.780255288002081 + "accuracy": 0.8218148946762085, + "time_taken": 4.0769241510006395 }, "3": { - "accuracy": 0.8863181471824646, - "time_taken": 4.911866419992293 + "accuracy": 0.8159054517745972, + "time_taken": 3.6375408430003517 }, "4": { + "accuracy": 0.8083934187889099, + "time_taken": 4.066575695999745 + } + }, + "1000": { + "0": { + "accuracy": 0.8711939454078674, + "time_taken": 5.933206661999975 + }, + "1": { + "accuracy": 0.8691906929016113, + "time_taken": 3.854292703000283 + }, + "2": { + "accuracy": 0.8723958730697632, + "time_taken": 4.351314984000055 + }, + "3": { + "accuracy": 0.8621795177459717, + "time_taken": 3.9070940849996987 + }, + "4": { + "accuracy": 0.8653846383094788, + "time_taken": 4.015361558999757 + } + }, + "5000": { + "0": { + "accuracy": 0.927584171295166, + "time_taken": 12.062748391000241 + }, + "1": { + "accuracy": 0.930588960647583, + "time_taken": 9.747504790999301 + }, + "2": { + "accuracy": 0.9271835088729858, + "time_taken": 8.544544992999363 + }, + "3": { + "accuracy": 0.9268830418586731, + "time_taken": 9.586745757999779 + }, + "4": { + "accuracy": 0.926682710647583, + "time_taken": 8.666436417999648 + } + } + }, + "Iterative Probabilistic Herding (cubic)": { + "25": { + "0": { + "accuracy": 0.5662263631820679, + "time_taken": 26.524820819000524 + }, + "1": { + "accuracy": 0.5555218458175659, + "time_taken": 22.971091108999644 + }, + "2": { + "accuracy": 0.5328129529953003, + "time_taken": 20.666025272000297 + }, + "3": { + "accuracy": 0.5929372310638428, + "time_taken": 18.04689451000013 + }, + "4": { + "accuracy": 0.5388152003288269, + "time_taken": 27.70253951199993 + } + }, + "50": { + "0": { + "accuracy": 0.6779000163078308, + "time_taken": 18.710263177000343 + }, + "1": { + "accuracy": 0.6948000192642212, + "time_taken": 8.480895946999226 + }, + "2": { + "accuracy": 0.6437996029853821, + "time_taken": 10.354708187000142 + }, + "3": { + "accuracy": 0.6868996024131775, + "time_taken": 12.53823441699933 + }, + "4": { + "accuracy": 0.6839997172355652, + "time_taken": 14.371656973999961 + } + }, + "100": { + "0": { + "accuracy": 0.779999852180481, + "time_taken": 9.07973868299996 + }, + "1": { + "accuracy": 0.7574002742767334, + "time_taken": 6.124219661000097 + }, + "2": { + "accuracy": 0.745400071144104, + "time_taken": 6.907441441000628 + }, + "3": { + "accuracy": 0.7519000172615051, + "time_taken": 6.637311533000684 + }, + "4": { + "accuracy": 0.7315998077392578, + "time_taken": 6.438722759000484 + } + }, + "500": { + "0": { + "accuracy": 0.859375, + "time_taken": 8.543036389000008 + }, + "1": { + "accuracy": 0.8518629670143127, + "time_taken": 4.909884863000116 + }, + "2": { + "accuracy": 0.8504607677459717, + "time_taken": 5.362902012000632 + }, + "3": { + "accuracy": 0.8542668223381042, + "time_taken": 5.555552781999722 + }, + "4": { + "accuracy": 0.8544671535491943, + "time_taken": 4.767543308999848 + } + }, + "1000": { + "0": { + "accuracy": 0.8869190812110901, + "time_taken": 9.989208918000259 + }, + "1": { + "accuracy": 0.8870192170143127, + "time_taken": 5.912931406999633 + }, + "2": { + "accuracy": 0.8907251954078674, + "time_taken": 6.551261347999571 + }, + "3": { "accuracy": 0.8823117017745972, - "time_taken": 5.024809098991682 + "time_taken": 6.210722205999446 + }, + "4": { + "accuracy": 0.8866186141967773, + "time_taken": 6.336088992000441 } }, "5000": { "0": { - "accuracy": 0.922776460647583, - "time_taken": 23.844807379995473 + "accuracy": 0.9264823794364929, + "time_taken": 31.977671014000407 }, "1": { - "accuracy": 0.9272836446762085, - "time_taken": 17.11210143000062 + "accuracy": 0.9283854365348816, + "time_taken": 24.778856055999313 }, "2": { - "accuracy": 0.9250801205635071, - "time_taken": 16.5623230929923 + "accuracy": 0.9249799847602844, + "time_taken": 24.93020751200038 }, "3": { - "accuracy": 0.9291867017745972, - "time_taken": 17.20165599399479 + "accuracy": 0.9258814454078674, + "time_taken": 23.563160845999846 }, "4": { - "accuracy": 0.9233773946762085, - "time_taken": 15.688548125996022 + "accuracy": 0.9239783883094788, + "time_taken": 23.45357099100056 } } } -} +} \ No newline at end of file diff --git a/benchmark/mnist_time_results.json b/benchmark/mnist_time_results.json index 1af9417d..1fa0ccbf 100644 --- a/benchmark/mnist_time_results.json +++ b/benchmark/mnist_time_results.json @@ -1,261 +1,335 @@ { "Random Sample": { "25": { - "0": 0.7094588610000301, - "1": 0.0017291089999389442, - "2": 0.001685944999962885, - "3": 0.001793641000176649, - "4": 0.0018410610000501038 + "0": 0.647310689000733, + "1": 0.001746535999700427, + "2": 0.0016490729976794682, + "3": 0.0017102239980886225, + "4": 0.0017371739995724056 }, "50": { - "0": 0.6013920729999427, - "1": 0.0017869449999352582, - "2": 0.0019502580000789749, - "3": 0.0016377110000576067, - "4": 0.0017066830000658229 + "0": 0.6191554599972733, + "1": 0.0017478970003139693, + "2": 0.001827262000006158, + "3": 0.0016880760013009422, + "4": 0.0017338979996566195 }, "100": { - "0": 0.6137222579999388, - "1": 0.0016203530000211686, - "2": 0.0018824950000180252, - "3": 0.001643035999904896, - "4": 0.001717244000019491 + "0": 0.6146928409980319, + "1": 0.0015965790007612668, + "2": 0.001910418999614194, + "3": 0.0017455419983889442, + "4": 0.0016654669998388272 }, "500": { - "0": 0.6200576559999718, - "1": 0.001770888000010018, - "2": 0.0017450970001391397, - "3": 0.0016370069999993575, - "4": 0.001675381000040943 + "0": 0.6247970969998278, + "1": 0.001661883001361275, + "2": 0.0020321159972809255, + "3": 0.001670051999099087, + "4": 0.0016311489998770412 }, "1000": { - "0": 0.6196925570000076, - "1": 0.002092210000000705, - "2": 0.0017158939999717404, - "3": 0.0015659470000173314, - "4": 0.001732565000111208 + "0": 0.6269827109972539, + "1": 0.0016431939984613564, + "2": 0.0016867839985934552, + "3": 0.0017590919997019228, + "4": 0.0016369789991586003 } }, "RP Cholesky": { "25": { - "0": 1.5147350790000473, - "1": 0.009266948999993474, - "2": 0.009186939999835886, - "3": 0.008905391000098462, - "4": 0.008803853000017625 + "0": 1.5792628960007278, + "1": 0.009678587997768773, + "2": 0.009231661999365315, + "3": 0.008769658001256175, + "4": 0.009058035000634845 }, "50": { - "0": 1.4470037309999952, - "1": 0.01754613100001734, - "2": 0.017912022000018624, - "3": 0.017566263000162508, - "4": 0.017581992000032187 + "0": 1.511850119997689, + "1": 0.017149059000075795, + "2": 0.01731069299785304, + "3": 0.016735487999540055, + "4": 0.017396160001226235 }, "100": { - "0": 1.4746420580000859, - "1": 0.03754237800001192, - "2": 0.03769687899989549, - "3": 0.037440069000012954, - "4": 0.03766440699996565 + "0": 1.5369271290001052, + "1": 0.03710502700050711, + "2": 0.036878171002172166, + "3": 0.03643120099877706, + "4": 0.0368910990000586 }, "500": { - "0": 2.1145162100000334, - "1": 0.39179865499988864, - "2": 0.3916123300000436, - "3": 0.39198892800004614, - "4": 0.39262377000000015 + "0": 2.0553133670000534, + "1": 0.38911631099836086, + "2": 0.3895299329997215, + "3": 0.3905363150006451, + "4": 0.3903145599979325 }, "1000": { - "0": 2.795978721000097, - "1": 1.2802431059999435, - "2": 1.2806992569999238, - "3": 1.2820121699999163, - "4": 1.2820419469999251 + "0": 2.8241980749990034, + "1": 1.2761649700005364, + "2": 1.2760263800009852, + "3": 1.2775282250004238, + "4": 1.2765133399989281 } }, "Kernel Herding": { "25": { - "0": 2.610353960999987, - "1": 0.28118345199993655, - "2": 0.285841161999997, - "3": 0.27556166899989876, - "4": 0.2756238229999326 + "0": 2.6031983829998353, + "1": 0.26574823800183367, + "2": 0.26442587800192996, + "3": 0.26058913100132486, + "4": 0.25983492900195415 }, "50": { - "0": 1.5984972850000077, - "1": 0.28247100600003705, - "2": 0.288643810999929, - "3": 0.28012494099994, - "4": 0.2785548000001654 + "0": 1.9571695459999319, + "1": 0.2680527449992951, + "2": 0.26516477200129884, + "3": 0.2671307109994814, + "4": 0.2625435150002886 }, "100": { - "0": 1.9940642040000967, - "1": 0.2899575900000855, - "2": 0.2972997360000136, - "3": 0.2834464029999708, - "4": 0.28465508800013595 + "0": 1.6518758749989502, + "1": 0.2736544399995182, + "2": 0.27683376699860673, + "3": 0.27000973599933786, + "4": 0.26825259999895934 }, "500": { - "0": 1.7415117790000068, - "1": 0.33662594000008994, - "2": 0.3445417009997982, - "3": 0.33440655700019306, - "4": 0.33455426499995156 + "0": 1.8148316490005527, + "1": 0.3298745380016044, + "2": 0.3237015800004883, + "3": 0.3160861960022885, + "4": 0.32276979100061 }, "1000": { - "0": 1.8535430869999345, - "1": 0.3871720610000011, - "2": 0.3925284009999359, - "3": 0.38465819700013526, - "4": 0.3837121839999327 + "0": 1.8868193730013445, + "1": 0.38048298899957445, + "2": 0.3765919639990898, + "3": 0.3759127279990935, + "4": 0.37570304299879353 } }, "Stein Thinning": { "25": { - "0": 3.8074047820000487, - "1": 3.824364295999999, - "2": 3.6565696080001544, - "3": 3.6072808989999885, - "4": 3.6074516559999665 + "0": 4.076632080999843, + "1": 4.0155544619992725, + "2": 3.9595811019971734, + "3": 3.7776183349997154, + "4": 3.8290312989993254 }, "50": { - "0": 4.19630774999996, - "1": 3.923172228999988, - "2": 3.862083046999942, - "3": 3.7715675970000575, - "4": 3.7609794219999912 + "0": 4.454431094000029, + "1": 4.158743233001587, + "2": 4.219952594998176, + "3": 3.934264821000397, + "4": 3.981639203000668 }, "100": { - "0": 3.9830489679999346, - "1": 4.064505925999924, - "2": 4.000870295999903, - "3": 3.931446998000183, - "4": 3.9275171780000164 + "0": 4.420032927999273, + "1": 4.288612154003204, + "2": 4.268337712001085, + "3": 4.1098778509986005, + "4": 4.155323740000313 }, "500": { - "0": 5.851763790000064, - "1": 5.488356060000001, - "2": 5.444121625000207, - "3": 5.364347303999921, - "4": 5.428865189000135 + "0": 5.637759639997967, + "1": 10.025119519999862, + "2": 5.700425971001096, + "3": 5.587590609000472, + "4": 5.601147779998428 }, "1000": { - "0": 6.939637783999956, - "1": 10.965542330000062, - "2": 6.903962380999928, - "3": 6.899562562000028, - "4": 6.997262595000166 + "0": 7.587774307998188, + "1": 7.324021808999532, + "2": 7.105633106002642, + "3": 7.04964620499959, + "4": 7.102810305997991 } }, "Kernel Thinning": { "25": { - "0": 66.63043982800002, - "1": 0.4887004889999389, - "2": 0.4734878320000462, - "3": 0.47251907500003654, - "4": 0.47228892800012545 + "0": 90.81835862099979, + "1": 0.4871493360005843, + "2": 0.47238080099850777, + "3": 0.4738097219997144, + "4": 0.4770787749985175 }, "50": { - "0": 33.34872766800004, - "1": 0.5037238669999624, - "2": 0.488602832999959, - "3": 0.4856810139999652, - "4": 0.48470954399999755 + "0": 46.26972372800083, + "1": 0.4933236199976818, + "2": 0.47538955799973337, + "3": 0.479368959000567, + "4": 0.47749236800154904 }, "100": { - "0": 21.044754845999933, - "1": 0.5149621710000929, - "2": 0.5004036840000481, - "3": 0.49941481700011536, - "4": 0.49638310099999217 + "0": 28.12213393500133, + "1": 0.5136646639984974, + "2": 0.491243342999951, + "3": 0.496500807999837, + "4": 0.4881571599980816 }, "500": { - "0": 7.190681866999967, - "1": 0.5032470499999135, - "2": 0.48689451700010977, - "3": 0.48538076199997704, - "4": 0.48296712800015484 + "0": 8.31735664399821, + "1": 0.48968988500200794, + "2": 0.48004734400092275, + "3": 0.47180004799884045, + "4": 0.47870473399962066 }, "1000": { - "0": 6.903874932000008, - "1": 0.5668272540000316, - "2": 0.5528795729999274, - "3": 0.5505331360000127, - "4": 0.5481448209998234 + "0": 7.5886145939985, + "1": 0.5649518889986211, + "2": 0.5526147460004722, + "3": 0.549001772000338, + "4": 0.5572626430002856 } }, "Compress++": { "25": { - "0": 29.412537138999937, - "1": 0.05838682499995684, - "2": 0.059164918999840665, - "3": 0.0580135689999679, - "4": 0.057627395000054094 + "0": 37.30952391499886, + "1": 0.05918412000028184, + "2": 0.05895868099833024, + "3": 0.058486374000494834, + "4": 0.05868467999971472 }, "50": { - "0": 18.233435394000026, - "1": 0.06534597700010636, - "2": 0.06501679099983448, - "3": 0.06366876599986426, - "4": 0.06367448700007117 + "0": 22.428665295003157, + "1": 0.0645626670011552, + "2": 0.06443759800094995, + "3": 0.06283377699946868, + "4": 0.06370369399883202 }, "100": { - "0": 8.529528905000006, - "1": 0.0648159620000115, - "2": 0.06605522800009567, - "3": 0.06563545900007739, - "4": 0.06551302200000464 + "0": 10.690804505000415, + "1": 0.06473967900092248, + "2": 0.0656564070013701, + "3": 0.06673681599932024, + "4": 0.06645065200063982 }, "500": { - "0": 4.349865860000023, - "1": 0.0864839649999567, - "2": 0.08642254599999433, - "3": 0.08871303000000808, - "4": 0.08833459100014807 + "0": 4.838294804998441, + "1": 0.08471525499771815, + "2": 0.08456600299905404, + "3": 0.08648833999905037, + "4": 0.08701666200067848 }, "1000": { - "0": 3.6908661520000123, - "1": 0.10315516699995442, - "2": 0.1031596630000422, - "3": 0.1060016710000582, - "4": 0.10459229200000664 + "0": 3.9391429269999207, + "1": 0.10608179200062295, + "2": 0.10270820200094022, + "3": 0.1046263509997516, + "4": 0.10333565999826533 } }, - "Probabilistic Iterative Herding": { + "Iterative Probabilistic Herding (constant)": { "25": { - "0": 4.567380666000076, - "1": 0.3798184410001113, - "2": 0.35353611400000773, - "3": 0.3526079970001774, - "4": 0.35121752999998535 + "0": 3.2858039469974756, + "1": 0.3433665390002716, + "2": 0.31859386100040865, + "3": 0.31910103900008835, + "4": 0.3209825199992338 }, "50": { - "0": 4.6527845999999045, - "1": 0.41235254699995494, - "2": 0.3860566419998577, - "3": 0.38475866199996744, - "4": 0.38376728100001856 + "0": 3.3452633079978114, + "1": 0.36137995299941394, + "2": 0.35409107799932826, + "3": 0.3516577080008574, + "4": 0.3545336329989368 }, "100": { - "0": 4.8177640440000005, - "1": 0.47743541500005904, - "2": 0.45443064400001276, - "3": 0.45361023100008424, - "4": 0.4517581079999218 + "0": 3.4772456729988335, + "1": 0.419977902998653, + "2": 0.4046461109974189, + "3": 0.4071592389991565, + "4": 0.40611362399795325 }, "500": { - "0": 5.6638604600000235, - "1": 1.0731660229999989, - "2": 1.0509365340001295, - "3": 1.0473623349998888, - "4": 1.0499444979998316 + "0": 4.121649624001293, + "1": 0.9177702929991938, + "2": 0.9034675580005569, + "3": 0.905480092002108, + "4": 0.9054613059997791 }, "1000": { - "0": 6.847450321999986, - "1": 1.8655788040000516, - "2": 1.843780653000067, - "3": 1.8360947809999288, - "4": 1.834880068000075 + "0": 5.015538179999567, + "1": 1.5666278080025222, + "2": 1.555398291999154, + "3": 1.5604459620008129, + "4": 1.5613997779983038 + } + }, + "Iterative Herding": { + "25": { + "0": 1.8262478720025683, + "1": 0.2904930989971035, + "2": 0.2827128229982918, + "3": 0.2810959639973589, + "4": 0.2824249270015571 + }, + "50": { + "0": 1.8156351859979623, + "1": 0.30463867400249, + "2": 0.29666196499965736, + "3": 0.296801904001768, + "4": 0.2964241440022306 + }, + "100": { + "0": 1.874870039999223, + "1": 0.334557213998778, + "2": 0.325431799999933, + "3": 0.3274396750020969, + "4": 0.32587794700157247 + }, + "500": { + "0": 2.275343524001073, + "1": 0.5682799839996733, + "2": 0.5668382160001784, + "3": 0.5603331589991285, + "4": 0.5630835640004079 + }, + "1000": { + "0": 2.590097914002399, + "1": 0.8541516059995047, + "2": 0.8420595110001159, + "3": 0.8399910410007578, + "4": 0.8505241100028798 + } + }, + "Iterative Probabilistic Herding (cubic)": { + "25": { + "0": 3.205438458997378, + "1": 0.3528836830009823, + "2": 0.34116740399986156, + "3": 0.34536475899949437, + "4": 0.34625313600190566 + }, + "50": { + "0": 3.3181408650016238, + "1": 0.4046067210001638, + "2": 0.39647927599799004, + "3": 0.39693685399834067, + "4": 0.3962372599999071 + }, + "100": { + "0": 3.5156353169986687, + "1": 0.5167118350000237, + "2": 0.51060877800046, + "3": 0.5102541699998255, + "4": 0.509720803998789 + }, + "500": { + "0": 4.725280201000714, + "1": 1.5142447790021833, + "2": 1.500269970001682, + "3": 1.5034325390006416, + "4": 1.5038313849981932 + }, + "1000": { + "0": 6.223259660000622, + "1": 2.8354778319990146, + "2": 2.8134299510020355, + "3": 2.811016760999337, + "4": 2.8105186829998274 } } -} +} \ No newline at end of file diff --git a/benchmark/pounce/Compress++_coreset.gif b/benchmark/pounce/Compress++_coreset.gif new file mode 100644 index 00000000..730d1982 Binary files /dev/null and b/benchmark/pounce/Compress++_coreset.gif differ diff --git a/benchmark/pounce/Iterative Herding_coreset.gif b/benchmark/pounce/Iterative Herding_coreset.gif new file mode 100644 index 00000000..b75f84fc Binary files /dev/null and b/benchmark/pounce/Iterative Herding_coreset.gif differ diff --git a/benchmark/pounce/Iterative Probabilistic Herding (constant)_coreset.gif b/benchmark/pounce/Iterative Probabilistic Herding (constant)_coreset.gif new file mode 100644 index 00000000..35c5c0bf Binary files /dev/null and b/benchmark/pounce/Iterative Probabilistic Herding (constant)_coreset.gif differ diff --git a/benchmark/pounce/Iterative Probabilistic Herding (cubic)_coreset.gif b/benchmark/pounce/Iterative Probabilistic Herding (cubic)_coreset.gif new file mode 100644 index 00000000..7c3feb3d Binary files /dev/null and b/benchmark/pounce/Iterative Probabilistic Herding (cubic)_coreset.gif differ diff --git a/benchmark/pounce/Kernel Herding_coreset.gif b/benchmark/pounce/Kernel Herding_coreset.gif new file mode 100644 index 00000000..631040a5 Binary files /dev/null and b/benchmark/pounce/Kernel Herding_coreset.gif differ diff --git a/benchmark/pounce/Kernel Thinning_coreset.gif b/benchmark/pounce/Kernel Thinning_coreset.gif new file mode 100644 index 00000000..cb47451e Binary files /dev/null and b/benchmark/pounce/Kernel Thinning_coreset.gif differ diff --git a/benchmark/pounce/RP Cholesky_coreset.gif b/benchmark/pounce/RP Cholesky_coreset.gif new file mode 100644 index 00000000..1bfdfc28 Binary files /dev/null and b/benchmark/pounce/RP Cholesky_coreset.gif differ diff --git a/benchmark/pounce/Random Sample_coreset.gif b/benchmark/pounce/Random Sample_coreset.gif new file mode 100644 index 00000000..bdb507ca Binary files /dev/null and b/benchmark/pounce/Random Sample_coreset.gif differ diff --git a/benchmark/pounce/Stein Thinning_coreset.gif b/benchmark/pounce/Stein Thinning_coreset.gif new file mode 100644 index 00000000..e482f750 Binary files /dev/null and b/benchmark/pounce/Stein Thinning_coreset.gif differ