From 126d73214183c80ffbb25fb73532699a349ed094 Mon Sep 17 00:00:00 2001 From: KyJah Keys Date: Fri, 27 Sep 2024 09:50:05 -0400 Subject: [PATCH 01/12] pinned delta rs release --- src/DeltaLake/Bridge/Cargo.lock | 1569 +++++++++++++++++------------ src/DeltaLake/Bridge/Cargo.toml | 4 +- src/DeltaLake/Bridge/src/error.rs | 5 + src/DeltaLake/Bridge/src/sql.rs | 51 +- src/DeltaLake/Bridge/src/table.rs | 22 +- src/DeltaLake/DeltaLake.csproj | 2 + 6 files changed, 1004 insertions(+), 649 deletions(-) diff --git a/src/DeltaLake/Bridge/Cargo.lock b/src/DeltaLake/Bridge/Cargo.lock index c94c550..fe3f700 100644 --- a/src/DeltaLake/Bridge/Cargo.lock +++ b/src/DeltaLake/Bridge/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "ahash" @@ -78,27 +78,27 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.82" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "arrayref" -version = "0.3.7" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa285343fba4d829d49985bdc541e3789cf6000ed0e84be7c039438df4a4e78c" +checksum = "05048a8932648b63f21c37d88b552ccc8a65afb6dfe9fc9f30ce79174c2e7a85" dependencies = [ "arrow-arith", "arrow-array", @@ -117,9 +117,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "753abd0a5290c1bcade7c6623a556f7d1659c5f4148b140b5b63ce7bd1a45705" +checksum = "1d8a57966e43bfe9a3277984a14c24ec617ad874e4c0e1d2a1b083a39cfbf22c" dependencies = [ "arrow-array", "arrow-buffer", @@ -132,9 +132,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" +checksum = "16f4a9468c882dc66862cef4e1fd8423d47e67972377d85d80e022786427768c" dependencies = [ "ahash", "arrow-buffer", @@ -143,15 +143,15 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "num", ] [[package]] name = "arrow-buffer" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4" +checksum = "c975484888fc95ec4a632cdc98be39c085b1bb518531b0c80c5d462063e5daa1" dependencies = [ "bytes", "half", @@ -160,28 +160,30 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e448e5dd2f4113bf5b74a1f26531708f5edcacc77335b7066f9398f4bcf4cdef" +checksum = "da26719e76b81d8bc3faad1d4dbdc1bcc10d14704e63dc17fc9f3e7e1e567c8e" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", - "base64 0.21.7", + "atoi", + "base64 0.22.1", "chrono", "comfy-table", "half", "lexical-core", "num", + "ryu", ] [[package]] name = "arrow-csv" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46af72211f0712612f5b18325530b9ad1bfbdc87290d5fbfd32a7da128983781" +checksum = "c13c36dc5ddf8c128df19bab27898eea64bf9da2b555ec1cd17a8ff57fba9ec2" dependencies = [ "arrow-array", "arrow-buffer", @@ -198,9 +200,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e" +checksum = "dd9d6f18c65ef7a2573ab498c374d8ae364b4a4edf67105357491c031f716ca5" dependencies = [ "arrow-buffer", "arrow-schema", @@ -210,9 +212,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03dea5e79b48de6c2e04f03f62b0afea7105be7b77d134f6c5414868feefb80d" +checksum = "e786e1cdd952205d9a8afc69397b317cfbb6e0095e445c69cda7e8da5c1eeb0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -225,9 +227,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8950719280397a47d37ac01492e3506a8a724b3fb81001900b866637a829ee0f" +checksum = "fb22284c5a2a01d73cebfd88a33511a3234ab45d66086b2ca2d1228c3498e445" dependencies = [ "arrow-array", "arrow-buffer", @@ -236,7 +238,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.2.6", + "indexmap 2.5.0", "lexical-core", "num", "serde", @@ -245,9 +247,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ed9630979034077982d8e74a942b7ac228f33dd93a93b615b4d02ad60c260be" +checksum = "42745f86b1ab99ef96d1c0bcf49180848a64fe2c7a7a0d945bc64fa2b21ba9bc" dependencies = [ "arrow-array", "arrow-buffer", @@ -260,9 +262,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf" +checksum = "4cd09a518c602a55bd406bcc291a967b284cfa7a63edfbf8b897ea4748aad23c" dependencies = [ "ahash", "arrow-array", @@ -270,24 +272,23 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown 0.14.3", ] [[package]] name = "arrow-schema" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029" +checksum = "9e972cd1ff4a4ccd22f86d3e53e835c2ed92e0eea6a3e8eadb72b4f1ac802cf8" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "serde", ] [[package]] name = "arrow-select" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" +checksum = "600bae05d43483d216fb3494f8c32fdbefd8aa4e1de237e790dbb3d9f44690a3" dependencies = [ "ahash", "arrow-array", @@ -299,15 +300,16 @@ dependencies = [ [[package]] name = "arrow-string" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f3b37f2aeece31a2636d1b037dabb69ef590e03bdc7eb68519b51ec86932a7" +checksum = "f0dc1985b67cb45f6606a248ac2b4a288849f196bab8c657ea5589f47cdd55e6" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "memchr", "num", "regex", "regex-syntax", @@ -315,9 +317,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.8" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60" +checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" dependencies = [ "bzip2", "flate2", @@ -333,26 +335,41 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.77", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" -version = "1.2.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.2.0" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2a89e0000cde82447155d64eeb71720b933b4396a6fbbebad3f8b4f88ca7b54" +checksum = "848d7b9b605720989929279fa644ce8f244d0ce3146fcca5b70e4eb7b3c020fc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -370,7 +387,6 @@ dependencies = [ "fastrand", "hex", "http 0.2.12", - "hyper", "ring", "time", "tokio", @@ -381,9 +397,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16838e6c9e12125face1c1eff1343c75e3ff540de98ff7ebd61874a89bcfeb9" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -393,14 +409,15 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.2.0" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4963ac9ff2d33a4231b3806c1c69f578f221a9cabb89ad2bde62ce2b442c8a7" +checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", "aws-smithy-http", + "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", @@ -408,6 +425,7 @@ dependencies = [ "fastrand", "http 0.2.12", "http-body 0.4.6", + "once_cell", "percent-encoding", "pin-project-lite", "tracing", @@ -416,9 +434,9 @@ dependencies = [ [[package]] name = "aws-sdk-dynamodb" -version = "1.22.0" +version = "1.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8c7e152b0c763cf1962a8e9fa4c32f1a646f9342b255bc657ce2f50c56dcc4" +checksum = "8abc61e7374a01ecebcc3fd465655a2e1b83455f15b26716bfac05c35d25fa1b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -439,9 +457,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.20.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32fcc572fd5c58489ec205ec3e4e5f7d63018898a485cbf922a462af496bc300" +checksum = "70a9d27ed1c12b1140c47daf1bc541606c43fdafd918c4797d520db0043ceef2" dependencies = [ "aws-credential-types", "aws-runtime", @@ -461,9 +479,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.20.0" +version = "1.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b6275fa8684a1192754221173b1f7a7c1260d6b0571cc2b8af09468eb0cffe5" +checksum = "44514a6ca967686cde1e2a1b81df6ef1883d0e3e570da8d8bc5c491dcb6fc29b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -483,9 +501,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.20.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30acd58272fd567e4853c5075d838be1626b59057e0249c9be5a1a7eb13bf70f" +checksum = "cd7a4d279762a35b9df97209f6808b95d4fe78547fe2316b4d200a0283960c5a" dependencies = [ "aws-credential-types", "aws-runtime", @@ -506,9 +524,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.0" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d6f29688a4be9895c0ba8bef861ad0c0dac5c15e9618b9b7a6c233990fc263" +checksum = "cc8db6904450bafe7473c6ca9123f88cc11089e41a025408f992db4e22d3be68" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -540,9 +558,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.7" +version = "0.60.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f10fa66956f01540051b0aa7ad54574640f748f9839e843442d99b970d3aff9" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -579,9 +597,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.3.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de34bcfa1fb3c82a80e252a753db34a6658e07f23d3a5b3fc96919518fa7a3f5" +checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -589,25 +607,26 @@ dependencies = [ "aws-smithy-types", "bytes", "fastrand", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "http-body 1.0.0", - "hyper", - "hyper-rustls", + "http-body 1.0.1", + "httparse", + "hyper 0.14.30", + "hyper-rustls 0.24.2", "once_cell", "pin-project-lite", "pin-utils", - "rustls", + "rustls 0.21.12", "tokio", "tracing", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.4.0" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc56a5c96ec741de6c5e6bf1ce6948be969d6506dfa9c39cffc284e31e4979b" +checksum = "e086682a53d3aa241192aa110fa8dfce98f2f5ac2ead0de84d41582c7e8fdb96" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -622,9 +641,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.1.8" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe14dceea1e70101d38fbf2a99e6a34159477c0fb95e68e05c66bd7ae4c3729" +checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" dependencies = [ "base64-simd", "bytes", @@ -633,7 +652,7 @@ dependencies = [ "http 0.2.12", "http 1.1.0", "http-body 0.4.6", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "itoa", "num-integer", @@ -648,55 +667,50 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.7" +version = "0.60.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872c68cf019c0e4afc5de7753c4f7288ce4b71663212771bf5e4542eb9346ca9" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.2.0" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a43b56df2c529fe44cb4d92bd64d0479883fb9608ff62daede4df5405381814" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "http 0.2.12", "rustc_version", "tracing", ] [[package]] -name = "backoff" -version = "0.4.0" +name = "backon" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +checksum = "e4fa97bb310c33c811334143cf64c5bb2b7b3c06e453db6b095d7061eff8f113" dependencies = [ - "futures-core", - "getrandom", - "instant", - "pin-project-lite", - "rand", + "fastrand", "tokio", ] [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -707,9 +721,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64-simd" @@ -729,9 +743,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "blake2" @@ -744,9 +758,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.1" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" dependencies = [ "arrayref", "arrayvec", @@ -766,9 +780,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.5.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -777,9 +791,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.1" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -793,9 +807,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.15.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" +checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" [[package]] name = "byteorder" @@ -805,9 +819,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "bytes-utils" @@ -846,7 +860,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da6bc11b07529f16944307272d5bd9b22530bc7d05751717c9d416586cedab49" dependencies = [ - "heck", + "heck 0.4.1", "indexmap 1.9.3", "log", "proc-macro2", @@ -860,12 +874,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.94" +version = "1.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" +checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -886,14 +901,14 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.5", + "windows-targets", ] [[package]] name = "chrono-tz" -version = "0.8.6" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" dependencies = [ "chrono", "chrono-tz-build", @@ -902,9 +917,9 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" dependencies = [ "parse-zoneinfo", "phf", @@ -944,9 +959,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "core-foundation" @@ -960,28 +975,34 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crunchy" version = "0.2.2" @@ -1021,12 +1042,13 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.5.3" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "crossbeam-utils", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -1034,9 +1056,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "36.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b360b692bf6c6d6e6b6dbaf41a3be0020daeceac0f406aed54c75331e50dbb" +checksum = "e4fd4a99fc70d40ef7e52b243b4a399c3f8d353a40d5ecb200deee05e49c61bb" dependencies = [ "ahash", "arrow", @@ -1049,30 +1071,36 @@ dependencies = [ "bzip2", "chrono", "dashmap", + "datafusion-catalog", "datafusion-common", + "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", "datafusion-functions", - "datafusion-functions-array", + "datafusion-functions-aggregate", + "datafusion-functions-nested", "datafusion-optimizer", "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-sql", "flate2", "futures", "glob", "half", - "hashbrown 0.14.3", - "indexmap 2.2.6", - "itertools", + "hashbrown 0.14.5", + "indexmap 2.5.0", + "itertools 0.12.1", "log", "num_cpus", "object_store", "parking_lot", "parquet", + "paste", "pin-project-lite", "rand", - "sqlparser 0.43.1", + "sqlparser 0.49.0", "tempfile", "tokio", "tokio-util", @@ -1082,11 +1110,25 @@ dependencies = [ "zstd", ] +[[package]] +name = "datafusion-catalog" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b3cfbd84c6003594ae1972314e3df303a27ce8ce755fcea3240c90f4c0529" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", +] + [[package]] name = "datafusion-common" -version = "36.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37f343ccc298f440e25aa38ff82678291a7acc24061c7370ba6c0ff5cc811412" +checksum = "44fdbc877e3e40dcf88cc8f283d9f5c8851f0a3aa07fee657b1b75ac1ad49b9c" dependencies = [ "ahash", "arrow", @@ -1095,18 +1137,29 @@ dependencies = [ "arrow-schema", "chrono", "half", + "hashbrown 0.14.5", + "instant", "libc", "num_cpus", "object_store", "parquet", - "sqlparser 0.43.1", + "sqlparser 0.49.0", +] + +[[package]] +name = "datafusion-common-runtime" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7496d1f664179f6ce3a5cbef6566056ccaf3ea4aa72cc455f80e62c1dd86b1" +dependencies = [ + "tokio", ] [[package]] name = "datafusion-execution" -version = "36.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9c93043081487e335399a21ebf8295626367a647ac5cb87d41d18afad7d0f7" +checksum = "799e70968c815b611116951e3dd876aef04bf217da31b72eec01ee6a959336a1" dependencies = [ "arrow", "chrono", @@ -1114,7 +1167,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "log", "object_store", "parking_lot", @@ -1125,54 +1178,95 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "36.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e204d89909e678846b6a95f156aafc1ee5b36cb6c9e37ec2e1449b078a38c818" +checksum = "1c1841c409d9518c17971d15c9bae62e629eb937e6fb6c68cd32e9186f8b30d2" dependencies = [ "ahash", "arrow", "arrow-array", + "arrow-buffer", + "chrono", "datafusion-common", "paste", - "sqlparser 0.43.1", + "serde_json", + "sqlparser 0.49.0", "strum", "strum_macros", ] [[package]] name = "datafusion-functions" -version = "36.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98f1c73f7801b2b8ba2297b3ad78ffcf6c1fc6b8171f502987eb9ad5cb244ee7" +checksum = "a8e481cf34d2a444bd8fa09b65945f0ce83dc92df8665b761505b3d9f351bebb" dependencies = [ "arrow", - "base64 0.21.7", + "arrow-buffer", + "base64 0.22.1", + "blake2", + "blake3", + "chrono", "datafusion-common", "datafusion-execution", "datafusion-expr", + "hashbrown 0.14.5", "hex", + "itertools 0.12.1", + "log", + "md-5", + "rand", + "regex", + "sha2", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4ece19f73c02727e5e8654d79cd5652de371352c1df3c4ac3e419ecd6943fb" +dependencies = [ + "ahash", + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", "log", + "paste", + "sqlparser 0.49.0", ] [[package]] -name = "datafusion-functions-array" -version = "36.0.0" +name = "datafusion-functions-nested" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d16a0ddf2c991526f6ffe2f47a72c6da0b7354d6c32411dd20631fe2e38937" +checksum = "a1474552cc824e8c9c88177d454db5781d4b66757d4aca75719306b8343a5e8d" dependencies = [ "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "itertools 0.12.1", "log", "paste", + "rand", ] [[package]] name = "datafusion-optimizer" -version = "36.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ae27e07bf1f04d327be5c2a293470879801ab5535204dc3b16b062fda195496" +checksum = "791ff56f55608bc542d1ea7a68a64bdc86a9413f5a381d06a39fd49c2a3ab906" dependencies = [ "arrow", "async-trait", @@ -1180,17 +1274,19 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.3", - "itertools", + "hashbrown 0.14.5", + "indexmap 2.5.0", + "itertools 0.12.1", "log", + "paste", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "36.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde620cd9ef76a3bca9c754fb68854bd2349c49f55baf97e08001f9e967f6d6b" +checksum = "9a223962b3041304a3e20ed07a21d5de3d88d7e4e71ca192135db6d24e3365a4" dependencies = [ "ahash", "arrow", @@ -1199,87 +1295,127 @@ dependencies = [ "arrow-ord", "arrow-schema", "arrow-string", - "base64 0.21.7", - "blake2", - "blake3", + "base64 0.22.1", "chrono", "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-physical-expr-common", "half", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "hex", - "indexmap 2.2.6", - "itertools", + "indexmap 2.5.0", + "itertools 0.12.1", "log", - "md-5", "paste", "petgraph", - "rand", "regex", - "sha2", - "unicode-segmentation", - "uuid", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5e7d8532a1601cd916881db87a70b0a599900d23f3db2897d389032da53bc6" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr", + "hashbrown 0.14.5", + "rand", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb9c78f308e050f5004671039786a925c3fee83b90004e9fcfd328d7febdcc0" +dependencies = [ + "datafusion-common", + "datafusion-execution", + "datafusion-physical-expr", + "datafusion-physical-plan", ] [[package]] name = "datafusion-physical-plan" -version = "36.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a4c75fba9ea99d64b2246cbd2fcae2e6fc973e6616b1015237a616036506dd4" +checksum = "8d1116949432eb2d30f6362707e2846d942e491052a206f2ddcb42d08aea1ffe" dependencies = [ "ahash", "arrow", "arrow-array", "arrow-buffer", + "arrow-ord", "arrow-schema", "async-trait", "chrono", "datafusion-common", + "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate", "datafusion-physical-expr", + "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.14.3", - "indexmap 2.2.6", - "itertools", + "hashbrown 0.14.5", + "indexmap 2.5.0", + "itertools 0.12.1", "log", "once_cell", "parking_lot", "pin-project-lite", "rand", "tokio", - "uuid", ] [[package]] name = "datafusion-proto" -version = "36.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2292251d5441d88d92a90d1511d5a8c88759a6562ff38ac1711b1587e6bf19c4" +checksum = "cf1d25864c18178d0e51438648f5e0fa08417dbbc39b642c1752cbbb1013abf0" dependencies = [ "arrow", "chrono", "datafusion", "datafusion-common", "datafusion-expr", + "datafusion-proto-common", + "object_store", + "prost", +] + +[[package]] +name = "datafusion-proto-common" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a683253732334526b1cc5314a73a0f786803831f7e189ed3fe387ac50d7222" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", "object_store", "prost", ] [[package]] name = "datafusion-sql" -version = "36.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21474a95c3a62d113599d21b439fa15091b538bac06bd20be0bb2e7d22903c09" +checksum = "b45d0180711165fe94015d7c4123eb3e1cf5fb60b1506453200b8d1ce666bef0" dependencies = [ "arrow", + "arrow-array", "arrow-schema", "datafusion-common", "datafusion-expr", "log", - "sqlparser 0.43.1", + "regex", + "sqlparser 0.49.0", + "strum", ] [[package]] @@ -1297,10 +1433,56 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "delta_kernel" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa08a82239f51e6d3d249c38f0f5bf7c8a78b28587e1b466893c9eac84d252d8" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-cast", + "arrow-json", + "arrow-ord", + "arrow-schema", + "arrow-select", + "bytes", + "chrono", + "delta_kernel_derive", + "either", + "fix-hidden-lifetime-bug", + "indexmap 2.5.0", + "itertools 0.13.0", + "lazy_static", + "parquet", + "roaring", + "rustc_version", + "serde", + "serde_json", + "strum", + "thiserror", + "tracing", + "url", + "uuid", + "visibility", + "z85", +] + +[[package]] +name = "delta_kernel_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec5c4fb5b59b1bd55ed8ebcf941f27a327d600c19a4a4103546846c358be93ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "deltalake" -version = "0.17.1" -source = "git+https://github.com/delta-io/delta-rs.git?branch=main#aa8f4d5390b8aa730bac9f9ba6f17372881bb06f" +version = "0.20.0" +source = "git+https://github.com/delta-io/delta-rs.git?tag=rust-v0.20.0#b3b23be91b8eb8207473785d4b0c20c5d2c174b2" dependencies = [ "deltalake-aws", "deltalake-azure", @@ -1310,8 +1492,8 @@ dependencies = [ [[package]] name = "deltalake-aws" -version = "0.1.0" -source = "git+https://github.com/delta-io/delta-rs.git?branch=main#aa8f4d5390b8aa730bac9f9ba6f17372881bb06f" +version = "0.3.0" +source = "git+https://github.com/delta-io/delta-rs.git?tag=rust-v0.20.0#b3b23be91b8eb8207473785d4b0c20c5d2c174b2" dependencies = [ "async-trait", "aws-config", @@ -1319,8 +1501,9 @@ dependencies = [ "aws-sdk-dynamodb", "aws-sdk-sts", "aws-smithy-runtime-api", - "backoff", + "backon", "bytes", + "chrono", "deltalake-core", "futures", "lazy_static", @@ -1336,8 +1519,8 @@ dependencies = [ [[package]] name = "deltalake-azure" -version = "0.1.0" -source = "git+https://github.com/delta-io/delta-rs.git?branch=main#aa8f4d5390b8aa730bac9f9ba6f17372881bb06f" +version = "0.3.0" +source = "git+https://github.com/delta-io/delta-rs.git?tag=rust-v0.20.0#b3b23be91b8eb8207473785d4b0c20c5d2c174b2" dependencies = [ "async-trait", "bytes", @@ -1354,8 +1537,8 @@ dependencies = [ [[package]] name = "deltalake-core" -version = "0.17.1" -source = "git+https://github.com/delta-io/delta-rs.git?branch=main#aa8f4d5390b8aa730bac9f9ba6f17372881bb06f" +version = "0.20.0" +source = "git+https://github.com/delta-io/delta-rs.git?tag=rust-v0.20.0#b3b23be91b8eb8207473785d4b0c20c5d2c174b2" dependencies = [ "arrow", "arrow-arith", @@ -1377,16 +1560,19 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-functions", + "datafusion-functions-aggregate", "datafusion-physical-expr", + "datafusion-physical-plan", "datafusion-proto", "datafusion-sql", + "delta_kernel", "either", "errno", "fix-hidden-lifetime-bug", "futures", - "hashbrown 0.14.3", - "indexmap 2.2.6", - "itertools", + "hashbrown 0.14.5", + "indexmap 2.5.0", + "itertools 0.13.0", "lazy_static", "libc", "maplit", @@ -1404,19 +1590,20 @@ dependencies = [ "roaring", "serde", "serde_json", - "sqlparser 0.44.0", + "sqlparser 0.51.0", "thiserror", "tokio", "tracing", "url", + "urlencoding", "uuid", "z85", ] [[package]] name = "deltalake-gcp" -version = "0.1.0" -source = "git+https://github.com/delta-io/delta-rs.git?branch=main#aa8f4d5390b8aa730bac9f9ba6f17372881bb06f" +version = "0.4.0" +source = "git+https://github.com/delta-io/delta-rs.git?tag=rust-v0.20.0#b3b23be91b8eb8207473785d4b0c20c5d2c174b2" dependencies = [ "async-trait", "bytes", @@ -1459,18 +1646,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "either" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" - -[[package]] -name = "encoding_rs" -version = "0.8.34" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" -dependencies = [ - "cfg-if", -] +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "equivalent" @@ -1480,9 +1658,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1490,24 +1668,24 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.2" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "fix-hidden-lifetime-bug" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ae9c2016a663983d4e40a9ff967d6dcac59819672f0b47f2b17574e99c33c8" +checksum = "ab7b4994e93dd63050356bdde7d417591d1b348523638dc1c1f539f16e338d55" dependencies = [ "fix-hidden-lifetime-bug-proc_macros", ] [[package]] name = "fix-hidden-lifetime-bug-proc_macros" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4c81935e123ab0741c4c4f0d9b8377e5fb21d3de7e062fa4b1263b1fbcba1ea" +checksum = "e8f0de9daf465d763422866d0538f07be1596e05623e120b37b4f715f5585200" dependencies = [ "proc-macro2", "quote", @@ -1522,9 +1700,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "23.5.26" +version = "24.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -1532,9 +1710,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", "miniz_oxide", @@ -1611,7 +1789,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.77", ] [[package]] @@ -1656,9 +1834,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -1667,9 +1845,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" [[package]] name = "glob" @@ -1689,7 +1867,26 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.2.6", + "indexmap 2.5.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", + "indexmap 2.5.0", "slab", "tokio", "tokio-util", @@ -1715,9 +1912,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", @@ -1729,6 +1926,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.9" @@ -1785,9 +1988,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http 1.1.0", @@ -1795,22 +1998,22 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", - "futures-core", + "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "pin-project-lite", ] [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" [[package]] name = "httpdate" @@ -1826,15 +2029,15 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -1848,6 +2051,26 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.6", + "http 1.1.0", + "http-body 1.0.1", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -1856,23 +2079,60 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper", + "hyper 0.14.30", "log", - "rustls", - "rustls-native-certs", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", ] [[package]] -name = "iana-time-zone" -version = "0.1.60" +name = "hyper-rustls" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", + "futures-util", + "http 1.1.0", + "hyper 1.4.1", + "hyper-util", + "rustls 0.23.13", + "rustls-native-certs 0.8.0", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.0", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "hyper 1.4.1", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", "js-sys", "wasm-bindgen", "windows-core", @@ -1909,21 +2169,24 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -1934,9 +2197,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "itertools" @@ -1947,6 +2210,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -1955,27 +2227,27 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.30" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lexical-core" @@ -2043,9 +2315,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libm" @@ -2055,15 +2327,15 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -2071,9 +2343,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lz4_flex" @@ -2113,9 +2385,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "mime" @@ -2125,29 +2397,30 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] name = "mio" -version = "0.8.11" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ + "hermit-abi", "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "num" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135b08af27d103b0a51f2ae0f8632117b7b185ccf931445affa8df530576a41" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ "num-bigint", "num-complex", @@ -2159,20 +2432,19 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ - "autocfg", "num-integer", "num-traits", ] [[package]] name = "num-complex" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", ] @@ -2194,9 +2466,9 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", "num-integer", @@ -2205,11 +2477,10 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "autocfg", "num-bigint", "num-integer", "num-traits", @@ -2217,9 +2488,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", "libm", @@ -2237,27 +2508,27 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.9.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" +checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" dependencies = [ "async-trait", - "base64 0.21.7", + "base64 0.22.1", "bytes", "chrono", "futures", "humantime", - "hyper", - "itertools", + "hyper 1.4.1", + "itertools 0.13.0", "md-5", "parking_lot", "percent-encoding", @@ -2265,7 +2536,7 @@ dependencies = [ "rand", "reqwest", "ring", - "rustls-pemfile 2.1.2", + "rustls-pemfile 2.1.3", "serde", "serde_json", "snafu", @@ -2304,9 +2575,9 @@ checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -2314,22 +2585,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.5", + "windows-targets", ] [[package]] name = "parquet" -version = "50.0.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750" +checksum = "e977b9066b4d3b03555c22bdc442f3fadebd96a39111249113087d0edb2691cd" dependencies = [ "ahash", "arrow-array", @@ -2339,14 +2610,14 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.21.7", + "base64 0.22.1", "brotli", "bytes", "chrono", "flate2", "futures", "half", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "lz4_flex", "num", "num-bigint", @@ -2358,22 +2629,23 @@ dependencies = [ "tokio", "twox-hash", "zstd", + "zstd-sys", ] [[package]] name = "parse-zoneinfo" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" dependencies = [ "regex", ] [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "percent-encoding" @@ -2383,12 +2655,12 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.2.6", + "indexmap 2.5.0", ] [[package]] @@ -2443,9 +2715,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "powerfmt" @@ -2455,24 +2727,27 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "proc-macro2" -version = "1.0.80" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", "prost-derive", @@ -2480,32 +2755,80 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools", + "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.77", ] [[package]] name = "quick-xml" -version = "0.31.0" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quinn" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.13", + "socket2", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +dependencies = [ + "bytes", + "rand", + "ring", + "rustc-hash", + "rustls 0.23.13", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +dependencies = [ + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -2542,18 +2865,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.6.0", ] [[package]] name = "regex" -version = "1.10.4" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", @@ -2563,9 +2886,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", @@ -2574,32 +2897,33 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" -version = "0.11.27" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "bytes", - "encoding_rs", "futures-core", "futures-util", - "h2", - "http 0.2.12", - "http-body 0.4.6", - "hyper", - "hyper-rustls", + "h2 0.4.6", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.4.1", + "hyper-rustls 0.27.3", + "hyper-util", "ipnet", "js-sys", "log", @@ -2607,16 +2931,17 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", - "rustls-native-certs", - "rustls-pemfile 1.0.4", + "quinn", + "rustls 0.23.13", + "rustls-native-certs 0.7.3", + "rustls-pemfile 2.1.3", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", - "system-configuration", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "tokio-util", "tower-service", "url", @@ -2624,7 +2949,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "winreg", + "windows-registry", ] [[package]] @@ -2644,9 +2969,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.10.3" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1c77081a55300e016cb86f2864415b7518741879db925b8d488a0ee0d2da6bf" +checksum = "8f4b84ba6e838ceb47b41de5194a60244fac43d9fe03b71dbe8c5a201081d6d1" dependencies = [ "bytemuck", "byteorder", @@ -2654,26 +2979,32 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.32" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -2682,16 +3013,30 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.11" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.8", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" version = "0.6.3" @@ -2704,6 +3049,32 @@ dependencies = [ "security-framework", ] +[[package]] +name = "rustls-native-certs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe", + "rustls-pemfile 2.1.3", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" +dependencies = [ + "openssl-probe", + "rustls-pemfile 2.1.3", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -2715,19 +3086,19 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.4.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" @@ -2739,17 +3110,28 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "ryu" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "same-file" @@ -2762,11 +3144,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2787,11 +3169,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.10.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -2800,9 +3182,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.10.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -2810,9 +3192,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "seq-macro" @@ -2822,31 +3204,32 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.197" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.77", ] [[package]] name = "serde_json" -version = "1.0.116" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -2874,11 +3257,17 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] @@ -2920,7 +3309,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "syn 1.0.109", @@ -2934,9 +3323,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2950,9 +3339,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.43.1" +version = "0.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f95c4bae5aba7cd30bd506f7140026ade63cff5afd778af8854026f9606bf5d4" +checksum = "a4a404d0e14905361b918cb8afdb73605e25c1d5029312bd9785142dcb3aa49e" dependencies = [ "log", "sqlparser_derive", @@ -2960,9 +3349,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.44.0" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf9c7ff146298ffda83a200f8d5084f08dcee1edfc135fcc1d646a45d50ffd6" +checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7" dependencies = [ "log", ] @@ -2975,7 +3364,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.77", ] [[package]] @@ -2986,31 +3375,31 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strum" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", - "syn 2.0.59", + "syn 2.0.77", ] [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -3025,9 +3414,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.59" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -3036,61 +3425,44 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" dependencies = [ - "core-foundation-sys", - "libc", + "futures-core", ] [[package]] name = "tempfile" -version = "3.10.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.77", ] [[package]] @@ -3145,9 +3517,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -3160,32 +3532,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.77", ] [[package]] @@ -3194,22 +3565,32 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls 0.23.13", + "rustls-pki-types", "tokio", ] [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -3223,9 +3604,9 @@ dependencies = [ [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -3247,7 +3628,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.77", ] [[package]] @@ -3289,30 +3670,30 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "untrusted" @@ -3322,9 +3703,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", @@ -3339,9 +3720,9 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "uuid" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "getrandom", "serde", @@ -3349,9 +3730,20 @@ dependencies = [ [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] [[package]] name = "vsimd" @@ -3386,34 +3778,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.77", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ "cfg-if", "js-sys", @@ -3423,9 +3816,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3433,22 +3826,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.77", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-streams" @@ -3465,61 +3858,60 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", ] [[package]] -name = "winapi" -version = "0.3.9" +name = "winapi-util" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "windows-sys 0.59.0", ] [[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.6" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "winapi", + "windows-targets", ] [[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" +name = "windows-registry" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets", +] [[package]] -name = "windows-core" -version = "0.52.0" +name = "windows-result" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" dependencies = [ - "windows-targets 0.52.5", + "windows-targets", ] [[package]] -name = "windows-sys" -version = "0.48.0" +name = "windows-strings" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-targets 0.48.5", + "windows-result", + "windows-targets", ] [[package]] @@ -3528,139 +3920,81 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets", ] [[package]] -name = "windows-targets" -version = "0.48.5" +name = "windows-sys" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" - -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "xmlparser" @@ -3685,53 +4019,54 @@ checksum = "2a599daf1b507819c1121f0bf87fa37eb19daac6aff3aefefd4e6e2e0f2020fc" [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.77", ] [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.12+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" dependencies = [ "cc", "pkg-config", diff --git a/src/DeltaLake/Bridge/Cargo.toml b/src/DeltaLake/Bridge/Cargo.toml index 9546344..f67096b 100644 --- a/src/DeltaLake/Bridge/Cargo.toml +++ b/src/DeltaLake/Bridge/Cargo.toml @@ -8,10 +8,10 @@ name = "delta_rs_bridge" crate-type = ["cdylib", "rlib"] [dependencies] -arrow = { version = "50", features = ["ffi"]} +arrow = { version = "52", features = ["ffi"]} chrono = "0.4.31" # deltalake = "0.16.5" -deltalake = { git = "https://github.com/delta-io/delta-rs.git", branch = "main", features = ["azure", "gcs", "s3", "datafusion"] } +deltalake = { git = "https://github.com/delta-io/delta-rs.git", tag = "rust-v0.20.0", features = ["azure", "gcs", "s3", "datafusion"] } futures = "0.3" libc = "0.2" serde_json = "1.0.116" diff --git a/src/DeltaLake/Bridge/src/error.rs b/src/DeltaLake/Bridge/src/error.rs index 44a4b6a..816072c 100644 --- a/src/DeltaLake/Bridge/src/error.rs +++ b/src/DeltaLake/Bridge/src/error.rs @@ -129,6 +129,11 @@ impl DeltaTableError { deltalake::DeltaTableError::CommitValidation { source: _ } => { DeltaTableErrorCode::InvalidData } + deltalake::DeltaTableError::KernelError(..) => DeltaTableErrorCode::Kernel, + deltalake::DeltaTableError::NotInitializedWithFiles(_) => DeltaTableErrorCode::Generic, + deltalake::DeltaTableError::ChangeDataNotRecorded { .. } => DeltaTableErrorCode::Generic, + deltalake::DeltaTableError::ChangeDataNotEnabled { .. } => DeltaTableErrorCode::Generic, + deltalake::DeltaTableError::ChangeDataInvalidVersionRange { .. } => DeltaTableErrorCode::Generic, }; Self::new(_runtime, code, &error_string) diff --git a/src/DeltaLake/Bridge/src/sql.rs b/src/DeltaLake/Bridge/src/sql.rs index 2d56b71..0630b1a 100644 --- a/src/DeltaLake/Bridge/src/sql.rs +++ b/src/DeltaLake/Bridge/src/sql.rs @@ -24,9 +24,20 @@ macro_rules! make_update { } for assign in $assignments { - for col in assign.id { - $update = $update.update(col.to_string(), assign.value.to_string()); - } + match assign.target { + AssignmentTarget::ColumnName(object_name) => { + for col in object_name.0 { + $update = $update.update(col.to_string(), assign.value.to_string()); + } + }, + AssignmentTarget::Tuple(vec) => { + for item in vec { + for col in item.0 { + $update = $update.update(col.to_string(), assign.value.to_string()); + } + } + }, + }; } $update }}; @@ -241,7 +252,7 @@ pub enum Statement { } #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] -#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +// #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum MergeClause { MatchedUpdate { predicate: Option, @@ -265,15 +276,11 @@ pub fn extract_table_factor_alias(table: TableFactor) -> Option { TableFactor::Table { name, alias, - args: _, - with_hints: _, - version: _, - partitions: _, + .. } => alias.map(|a| a.to_string()).or(Some(name.to_string())), TableFactor::Derived { - lateral: _, - subquery: _, alias, + .. } => alias.map(|a| a.to_string()), TableFactor::TableFunction { expr: _, alias } => alias.map(|a| a.to_string()), TableFactor::Function { @@ -284,34 +291,28 @@ pub fn extract_table_factor_alias(table: TableFactor) -> Option { } => alias.map(|a| a.to_string()), TableFactor::UNNEST { alias, - array_exprs: _, - with_offset: _, - with_offset_alias: _, + .. } => alias.map(|a| a.to_string()), TableFactor::NestedJoin { - table_with_joins: _, alias, + .. } => alias.map(|a| a.to_string()), TableFactor::Pivot { - table: _, - aggregate_function: _, - value_column: _, - pivot_values: _, alias, + .. } => alias.map(|a| a.to_string()), TableFactor::Unpivot { - table: _, - value: _, - name: _, - columns: _, alias, + .. } => alias.map(|a| a.to_string()), TableFactor::JsonTable { - json_expr: _, - json_path: _, - columns: _, alias, + .. } => alias.map(|a| a.to_string()), + TableFactor::MatchRecognize { + alias, + .. + } => alias.map(|a| a.to_string()), } } diff --git a/src/DeltaLake/Bridge/src/table.rs b/src/DeltaLake/Bridge/src/table.rs index a9ddc44..e18011c 100644 --- a/src/DeltaLake/Bridge/src/table.rs +++ b/src/DeltaLake/Bridge/src/table.rs @@ -17,7 +17,7 @@ use deltalake::{ datafusion::{ dataframe::DataFrame, datasource::{MemTable, TableProvider}, - execution::context::{SQLOptions, SessionContext}, + execution::context::{SQLOptions, SessionContext}, sql::sqlparser::ast::AssignmentTarget, }, kernel::StructType, operations::{ @@ -973,9 +973,20 @@ pub extern "C" fn table_update( } for assign in assignments { - for col in assign.id { - ub = ub.with_update(col.to_string(), assign.value.to_string()); - } + match assign.target { + AssignmentTarget::ColumnName(object_name) => { + for col in object_name.0 { + ub = ub.with_update(col.to_string(), assign.value.to_string()); + } + }, + AssignmentTarget::Tuple(vec) => { + for item in vec { + for col in item.0 { + ub = ub.with_update(col.to_string(), assign.value.to_string()); + } + } + }, + }; } match ub.await { @@ -1554,7 +1565,7 @@ async fn create_delta_table( })?; let mut builder = DeltaOps(table) .create() - .with_columns(delta_schema.fields().clone()) + .with_columns(delta_schema.fields().cloned()) .with_save_mode(mode) .with_partition_columns(partition_by); if let Some(name) = &name { @@ -1636,6 +1647,7 @@ fn ffi_to_batches( Ok((read_batches, schema)) } + #[cfg(test)] mod tests { use crate::sql::{DeltaLakeParser, Statement}; diff --git a/src/DeltaLake/DeltaLake.csproj b/src/DeltaLake/DeltaLake.csproj index ac0358c..0033271 100644 --- a/src/DeltaLake/DeltaLake.csproj +++ b/src/DeltaLake/DeltaLake.csproj @@ -11,6 +11,8 @@ enable net6.0;net8.0; rc-1 + DeltaLake + Delta.Net From b17adfc4fbfd9a37b1d2e5dea14c057e83466056 Mon Sep 17 00:00:00 2001 From: KyJah Keys Date: Fri, 27 Sep 2024 10:01:24 -0400 Subject: [PATCH 02/12] updated artifact action --- .github/workflows/package.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index d871135..401a110 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -76,14 +76,14 @@ jobs: - name: Upload bridge library if: ${{ !matrix.alternative-target }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.out-prefix }}-bridge path: src/DeltaLake/Bridge/target/release/${{ matrix.out-file }} - name: Upload bridge library alternative target if: ${{ matrix.alternative-target != '' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.out-prefix }}-bridge path: src/DeltaLake/Bridge/target/${{ matrix.alternative-target }}/release/${{ matrix.out-file }} @@ -109,7 +109,7 @@ jobs: run: dotnet pack src/DeltaLake/DeltaLake.csproj -c Release /p:BridgeLibraryRoot=${{ github.workspace }}/bridge-libraries - name: Upload NuGet artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: nuget-package path: | From 4ec71f3d9239e1f3f84bd8b23536fc19244aff0d Mon Sep 17 00:00:00 2001 From: KyJah Keys Date: Fri, 27 Sep 2024 10:11:53 -0400 Subject: [PATCH 03/12] updated build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4fb2afb..c2fd862 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,7 @@ jobs: - name: Upload test failure if: ${{ failure() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: test-fail-${{ matrix.os }} path: tests/DeltaLake.Tests/TestResults From 1789ae5ad116706b50d8cccee2c3519b266a34c1 Mon Sep 17 00:00:00 2001 From: KyJah Keys Date: Fri, 27 Sep 2024 10:49:32 -0400 Subject: [PATCH 04/12] added RUST_BACKTRACE to get panic info --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c2fd862..60efdc3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,6 +62,8 @@ jobs: run: dotnet format --verify-no-changes - name: Test + env: + RUST_BACKTRACE: 1 run: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=lcov /p:CoverletOutput=./lcov.info --logger "console;verbosity=detailed" --blame-crash -v n - name: Upload test failure From 2a8d61113cb5d0f598a3cf80a50b950ba8570c1c Mon Sep 17 00:00:00 2001 From: KyJah Keys Date: Fri, 27 Sep 2024 14:43:27 -0400 Subject: [PATCH 05/12] refactored to make this work with latest deltalake --- README.md | 2 +- examples/local/Program.cs | 113 +- src/DeltaLake/Bridge/ByteArray.cs | 122 +- src/DeltaLake/Bridge/Runtime.cs | 372 ++--- src/DeltaLake/Bridge/Table.cs | 1348 +++++++++--------- src/DeltaLake/Bridge/src/error.rs | 4 + src/DeltaLake/Bridge/src/lib.rs | 1 + src/DeltaLake/Bridge/src/runtime.rs | 2 +- src/DeltaLake/Bridge/src/sql.rs | 14 + src/DeltaLake/Bridge/src/table.rs | 36 +- src/DeltaLake/Table/TableCreateOptions.cs | 138 +- src/DeltaLake/Table/TableMetadata.cs | 238 ++-- tests/DeltaLake.Tests/DeltaLake.Tests.csproj | 10 +- tests/DeltaLake.Tests/Table/TableHelpers.cs | 305 ++-- tests/DeltaLake.Tests/Table/TableTests.cs | 461 +++--- tests/DeltaLake.Tests/Table/UpdateTests.cs | 192 +-- 16 files changed, 1697 insertions(+), 1661 deletions(-) diff --git a/README.md b/README.md index 6e78456..12bf458 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ public static Task CreateDeltaTable( runtime, new TableCreateOptions(uri, schema) { - Configuration = new Dictionary(), + Configuration = new Dictionary(), }, cancellationToken); } diff --git a/examples/local/Program.cs b/examples/local/Program.cs index 4f01dc9..4769c28 100644 --- a/examples/local/Program.cs +++ b/examples/local/Program.cs @@ -1,58 +1,57 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Apache.Arrow; -using Apache.Arrow.Memory; -using Apache.Arrow.Types; -using DeltaLake.Runtime; -using DeltaLake.Table; - -namespace local; - -public class Program -{ - public static async Task Main(string[] args) - { - var uri = args[0]; - int length; - if (args.Length < 2 || !int.TryParse(args[1], out length)) - { - length = 10; - } - - var runtime = new DeltaRuntime(RuntimeOptions.Default); - { - var builder = new Apache.Arrow.Schema.Builder(); - builder.Field(fb => - { - fb.Name("test"); - fb.DataType(Int32Type.Default); - fb.Nullable(false); - }); - var schema = builder.Build(); - var allocator = new NativeMemoryAllocator(); - var recordBatchBuilder = new RecordBatch.Builder(allocator) - .Append("test", false, col => col.Int32(arr => arr.AppendRange(Enumerable.Range(0, length)))); - using var table = await DeltaTable.CreateAsync( - runtime, - new TableCreateOptions(uri, schema) - { - Configuration = new Dictionary - { - ["delta.dataSkippingNumIndexedCols"] = "32", - ["delta.setTransactionRetentionDuration"] = null, - } - }, - CancellationToken.None); - var options = new InsertOptions - { - SaveMode = SaveMode.Append, - }; - await table.InsertAsync([recordBatchBuilder.Build()], schema, options, CancellationToken.None); - } - - runtime.Dispose(); - } +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Apache.Arrow; +using Apache.Arrow.Memory; +using Apache.Arrow.Types; +using DeltaLake.Runtime; +using DeltaLake.Table; + +namespace local; + +public class Program +{ + public static async Task Main(string[] args) + { + var uri = args[0]; + int length; + if (args.Length < 2 || !int.TryParse(args[1], out length)) + { + length = 10; + } + + var runtime = new DeltaRuntime(RuntimeOptions.Default); + { + var builder = new Apache.Arrow.Schema.Builder(); + builder.Field(fb => + { + fb.Name("test"); + fb.DataType(Int32Type.Default); + fb.Nullable(false); + }); + var schema = builder.Build(); + var allocator = new NativeMemoryAllocator(); + var recordBatchBuilder = new RecordBatch.Builder(allocator) + .Append("test", false, col => col.Int32(arr => arr.AppendRange(Enumerable.Range(0, length)))); + using var table = await DeltaTable.CreateAsync( + runtime, + new TableCreateOptions(uri, schema) + { + Configuration = new Dictionary + { + ["delta.dataSkippingNumIndexedCols"] = "32", + } + }, + CancellationToken.None); + var options = new InsertOptions + { + SaveMode = SaveMode.Append, + }; + await table.InsertAsync([recordBatchBuilder.Build()], schema, options, CancellationToken.None); + } + + runtime.Dispose(); + } } \ No newline at end of file diff --git a/src/DeltaLake/Bridge/ByteArray.cs b/src/DeltaLake/Bridge/ByteArray.cs index 72dbe62..3189230 100644 --- a/src/DeltaLake/Bridge/ByteArray.cs +++ b/src/DeltaLake/Bridge/ByteArray.cs @@ -1,62 +1,62 @@ -using System; -using System.Runtime.InteropServices; - -namespace DeltaLake.Bridge -{ - /// - /// Representation of a byte array owned by Core. - /// - internal sealed class ByteArray : SafeHandle - { - private readonly Runtime runtime; - private readonly unsafe Interop.ByteArray* byteArray; - - /// - /// Initializes a new instance of the class. - /// - /// Runtime to use to free the byte array. - /// Byte array pointer. - public unsafe ByteArray(Runtime runtime, Interop.ByteArray* byteArray) - : base((IntPtr)byteArray, true) - { - this.runtime = runtime; - this.byteArray = byteArray; - } - - /// - public override unsafe bool IsInvalid => false; - - /// - /// Convert the byte array to a UTF8 string. - /// - /// Converted string. - public string ToUTF8() - { - unsafe - { - return ByteArrayRef.StrictUTF8.GetString(byteArray->data, (int)byteArray->size); - } - } - - /// - /// Copy the byte array to a new byte array. - /// - /// The new byte array. - public byte[] ToByteArray() - { - unsafe - { - var bytes = new byte[(int)byteArray->size]; - Marshal.Copy((IntPtr)byteArray->data, bytes, 0, (int)byteArray->size); - return bytes; - } - } - - /// - protected override unsafe bool ReleaseHandle() - { - runtime.FreeByteArray(byteArray); - return true; - } - } +using System; +using System.Runtime.InteropServices; + +namespace DeltaLake.Bridge +{ + /// + /// Representation of a byte array owned by Core. + /// + internal sealed class ByteArray : SafeHandle + { + private readonly Runtime runtime; + private readonly unsafe Interop.ByteArray* byteArray; + + /// + /// Initializes a new instance of the class. + /// + /// Runtime to use to free the byte array. + /// Byte array pointer. + public unsafe ByteArray(Runtime runtime, Interop.ByteArray* byteArray) + : base((IntPtr)byteArray, true) + { + this.runtime = runtime; + this.byteArray = byteArray; + } + + /// + public override unsafe bool IsInvalid => false; + + /// + /// Convert the byte array to a UTF8 string. + /// + /// Converted string. + public string ToUTF8() + { + unsafe + { + return ByteArrayRef.StrictUTF8.GetString(byteArray->data, (int)byteArray->size); + } + } + + /// + /// Copy the byte array to a new byte array. + /// + /// The new byte array. + public byte[] ToByteArray() + { + unsafe + { + var bytes = new byte[(int)byteArray->size]; + Marshal.Copy((IntPtr)byteArray->data, bytes, 0, (int)byteArray->size); + return bytes; + } + } + + /// + protected override unsafe bool ReleaseHandle() + { + runtime.FreeByteArray(byteArray); + return true; + } + } } \ No newline at end of file diff --git a/src/DeltaLake/Bridge/Runtime.cs b/src/DeltaLake/Bridge/Runtime.cs index 1c2e660..03933b0 100644 --- a/src/DeltaLake/Bridge/Runtime.cs +++ b/src/DeltaLake/Bridge/Runtime.cs @@ -1,187 +1,187 @@ -using System; -using System.Buffers; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.InteropServices; -using System.Threading.Tasks; -using Apache.Arrow.C; -using DeltaLake.Errors; - -namespace DeltaLake.Bridge -{ - /// - /// Core-owned runtime. - /// - internal sealed class Runtime : SafeHandle - { - /// - /// Initializes a new instance of the class. - /// - /// Runtime options. - /// Any internal core error. - public Runtime(DeltaLake.Runtime.RuntimeOptions options) - : base(IntPtr.Zero, true) - { - unsafe - { - var res = Interop.Methods.runtime_new(null); - // If it failed, copy byte array, free runtime and byte array. Otherwise just - // return runtime. - if (res.fail != null) - { - var message = ByteArrayRef.StrictUTF8.GetString( - res.fail->data, - (int)res.fail->size); - Interop.Methods.byte_array_free(res.runtime, res.fail); - Interop.Methods.runtime_free(res.runtime); - throw new InvalidOperationException(message); - } - Ptr = res.runtime; - SetHandle((IntPtr)Ptr); - } - } - - /// - public override unsafe bool IsInvalid => false; - - public async Task LoadTableAsync( - string tableUri, - DeltaLake.Table.TableOptions options, - System.Threading.CancellationToken cancellationToken) - { - var buffer = ArrayPool.Shared.Rent(System.Text.Encoding.UTF8.GetByteCount(tableUri)); - var encodedLength = System.Text.Encoding.UTF8.GetBytes(tableUri, buffer); - try - { - return await LoadTableAsync(buffer.AsMemory(0, encodedLength), options, cancellationToken).ConfigureAwait(false); - } - finally - { - ArrayPool.Shared.Return(buffer); - } - } - - internal async Task
LoadTableAsync( - Memory tableUri, - DeltaLake.Table.TableOptions options, - System.Threading.CancellationToken cancellationToken) - { - var tsc = new TaskCompletionSource
(); - using (var scope = new Scope()) - { - unsafe - { - var nativeOptions = new Interop.TableOptions() - { - version = options.Version.HasValue ? unchecked((long)options.Version.Value) : -1L, - without_files = (byte)(options.WithoutFiles ? 1 : 0), - log_buffer_size = options.LogBufferSize ?? (nuint)0, - storage_options = options.StorageOptions != null ? scope.Dictionary(this, options.StorageOptions) : null, - }; - Interop.Methods.table_new( - Ptr, - scope.Pointer(scope.ByteArray(tableUri)), - scope.Pointer(nativeOptions), - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((success, fail) => - { - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - return; - } - - if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(Ptr, fail)); - } - else - { - tsc.TrySetResult(new Table(this, success)); - } - })); - } - - return await tsc.Task.ConfigureAwait(false); - } - } - - internal async Task
CreateTableAsync(DeltaLake.Table.TableCreateOptions options, System.Threading.CancellationToken cancellationToken) - { - var tsc = new TaskCompletionSource
(); - using (var scope = new Scope()) - { - unsafe - { - var nativeSchema = CArrowSchema.Create(); - try - { - CArrowSchemaExporter.ExportSchema(options.Schema, nativeSchema); - var saveMode = Table.ConvertSaveMode(options.SaveMode); - var nativeOptions = new Interop.TableCreatOptions() - { - table_uri = scope.ByteArray(options.TableLocation), - schema = nativeSchema, - partition_by = scope.ArrayPointer(options.PartitionBy.Select(x => scope.ByteArray(x)).ToArray()), - partition_count = (nuint)options.PartitionBy.Count, - mode = saveMode.Ref, - name = scope.ByteArray(options.Name), - description = scope.ByteArray(options.Description), - configuration = scope.OptionalDictionary(this, options.Configuration ?? new Dictionary()), - custom_metadata = scope.Dictionary(this, options.CustomMetadata ?? new Dictionary()), - storage_options = scope.Dictionary(this, options.StorageOptions ?? new Dictionary()), - }; - Interop.Methods.create_deltalake( - Ptr, - scope.Pointer(nativeOptions), - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((success, fail) => - { - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - return; - } - - if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(Ptr, fail)); - } - else - { - tsc.TrySetResult(new Table(this, success)); - } - })); - } - finally - { - CArrowSchema.Free(nativeSchema); - } - } - - return await tsc.Task.ConfigureAwait(false); - } - } - - /// - /// Gets the pointer to the runtime. - /// - internal unsafe Interop.Runtime* Ptr { get; private init; } - - /// - /// Free a byte array. - /// - /// Byte array to free. - internal unsafe void FreeByteArray(Interop.ByteArray* byteArray) - { - Interop.Methods.byte_array_free(Ptr, byteArray); - } - - /// - protected override unsafe bool ReleaseHandle() - { - Interop.Methods.runtime_free(Ptr); - return true; - } - } +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Apache.Arrow.C; +using DeltaLake.Errors; + +namespace DeltaLake.Bridge +{ + /// + /// Core-owned runtime. + /// + internal sealed class Runtime : SafeHandle + { + /// + /// Initializes a new instance of the class. + /// + /// Runtime options. + /// Any internal core error. + public Runtime(DeltaLake.Runtime.RuntimeOptions options) + : base(IntPtr.Zero, true) + { + unsafe + { + var res = Interop.Methods.runtime_new(null); + // If it failed, copy byte array, free runtime and byte array. Otherwise just + // return runtime. + if (res.fail != null) + { + var message = ByteArrayRef.StrictUTF8.GetString( + res.fail->data, + (int)res.fail->size); + Interop.Methods.byte_array_free(res.runtime, res.fail); + Interop.Methods.runtime_free(res.runtime); + throw new InvalidOperationException(message); + } + Ptr = res.runtime; + SetHandle((IntPtr)Ptr); + } + } + + /// + public override unsafe bool IsInvalid => false; + + public async Task
LoadTableAsync( + string tableUri, + DeltaLake.Table.TableOptions options, + System.Threading.CancellationToken cancellationToken) + { + var buffer = ArrayPool.Shared.Rent(System.Text.Encoding.UTF8.GetByteCount(tableUri)); + var encodedLength = System.Text.Encoding.UTF8.GetBytes(tableUri, buffer); + try + { + return await LoadTableAsync(buffer.AsMemory(0, encodedLength), options, cancellationToken).ConfigureAwait(false); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + internal async Task
LoadTableAsync( + Memory tableUri, + DeltaLake.Table.TableOptions options, + System.Threading.CancellationToken cancellationToken) + { + var tsc = new TaskCompletionSource
(); + using (var scope = new Scope()) + { + unsafe + { + var nativeOptions = new Interop.TableOptions() + { + version = options.Version.HasValue ? unchecked((long)options.Version.Value) : -1L, + without_files = (byte)(options.WithoutFiles ? 1 : 0), + log_buffer_size = options.LogBufferSize ?? (nuint)0, + storage_options = options.StorageOptions != null ? scope.Dictionary(this, options.StorageOptions) : null, + }; + Interop.Methods.table_new( + Ptr, + scope.Pointer(scope.ByteArray(tableUri)), + scope.Pointer(nativeOptions), + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((success, fail) => + { + if (cancellationToken.IsCancellationRequested) + { + Task.Run(() => tsc.TrySetCanceled(cancellationToken));; + return; + } + + if (fail != null) + { + Task.Run(() => tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(Ptr, fail))); + } + else + { + Task.Run(() => tsc.TrySetResult(new Table(this, success))); + } + })); + } + + return await tsc.Task.ConfigureAwait(false); + } + } + + internal async Task
CreateTableAsync(DeltaLake.Table.TableCreateOptions options, System.Threading.CancellationToken cancellationToken) + { + var tsc = new TaskCompletionSource
(); + using (var scope = new Scope()) + { + unsafe + { + var nativeSchema = CArrowSchema.Create(); + try + { + CArrowSchemaExporter.ExportSchema(options.Schema, nativeSchema); + var saveMode = Table.ConvertSaveMode(options.SaveMode); + var nativeOptions = new Interop.TableCreatOptions() + { + table_uri = scope.ByteArray(options.TableLocation), + schema = nativeSchema, + partition_by = scope.ArrayPointer(options.PartitionBy.Select(x => scope.ByteArray(x)).ToArray()), + partition_count = (nuint)options.PartitionBy.Count, + mode = saveMode.Ref, + name = scope.ByteArray(options.Name), + description = scope.ByteArray(options.Description), + configuration = scope.OptionalDictionary(this, options.Configuration ?? new Dictionary()), + custom_metadata = scope.Dictionary(this, options.CustomMetadata ?? new Dictionary()), + storage_options = scope.Dictionary(this, options.StorageOptions ?? new Dictionary()), + }; + Interop.Methods.create_deltalake( + Ptr, + scope.Pointer(nativeOptions), + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((success, fail) => + { + if (cancellationToken.IsCancellationRequested) + { + Task.Run(() => tsc.TrySetCanceled(cancellationToken));; + return; + } + + if (fail != null) + { + Task.Run(() => tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(Ptr, fail))); + } + else + { + Task.Run(() => tsc.TrySetResult(new Table(this, success))); + } + })); + } + finally + { + CArrowSchema.Free(nativeSchema); + } + } + + return await tsc.Task.ConfigureAwait(false); + } + } + + /// + /// Gets the pointer to the runtime. + /// + internal unsafe Interop.Runtime* Ptr { get; private init; } + + /// + /// Free a byte array. + /// + /// Byte array to free. + internal unsafe void FreeByteArray(Interop.ByteArray* byteArray) + { + Interop.Methods.byte_array_free(Ptr, byteArray); + } + + /// + protected override unsafe bool ReleaseHandle() + { + Interop.Methods.runtime_free(Ptr); + return true; + } + } } \ No newline at end of file diff --git a/src/DeltaLake/Bridge/Table.cs b/src/DeltaLake/Bridge/Table.cs index 48f2ad1..f8bd841 100644 --- a/src/DeltaLake/Bridge/Table.cs +++ b/src/DeltaLake/Bridge/Table.cs @@ -1,672 +1,678 @@ -using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; -using System.Threading.Tasks; -using Apache.Arrow; -using Apache.Arrow.C; -using Apache.Arrow.Ipc; -using DeltaLake.Bridge.Interop; -using DeltaLake.Errors; -using DeltaLake.Table; -using ICancellationToken = System.Threading.CancellationToken; - -namespace DeltaLake.Bridge -{ - /// - /// Reference to unmanaged delta table - /// - internal sealed class Table : SafeHandle - { - internal static readonly ByteArrayRef SaveModeAppend = ByteArrayRef.FromUTF8("append"); - - internal static readonly ByteArrayRef SaveModeOverwrite = ByteArrayRef.FromUTF8("overwrite"); - - internal static readonly ByteArrayRef SaveModeError = ByteArrayRef.FromUTF8("error"); - - internal static readonly ByteArrayRef SaveModeIfgnore = ByteArrayRef.FromUTF8("ignore"); - - private readonly unsafe Interop.RawDeltaTable* _ptr; - - private readonly Runtime _runtime; - - /// - /// Creates a table - /// - internal unsafe Table(Runtime runtime, Interop.RawDeltaTable* inner) - : base(IntPtr.Zero, true) - { - _ptr = inner; - _runtime = runtime; - SetHandle((IntPtr)_ptr); - } - - /// - public override bool IsInvalid => false; - - /// - /// Returns the current version of the table - /// - /// - public async Task LoadVersionAsync(ulong version, ICancellationToken cancellationToken) - { - var tsc = new TaskCompletionSource(); - using (var scope = new Scope()) - { - unsafe - { - Interop.Methods.table_load_version( - _runtime.Ptr, - _ptr, - unchecked((long)version), - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((fail) => - { - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - return; - } - - if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); - } - else - { - tsc.TrySetResult(true); - } - })); - } - - await tsc.Task.ConfigureAwait(false); - } - } - - internal async Task LoadTimestampAsync(long timestampMilliseconds, ICancellationToken cancellationToken) - { - var tsc = new TaskCompletionSource(); - using (var scope = new Scope()) - { - unsafe - { - Interop.Methods.table_load_with_datetime( - _runtime.Ptr, - _ptr, - timestampMilliseconds, - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((fail) => - { - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - return; - } - - if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); - } - else - { - tsc.TrySetResult(true); - } - })); - } - - await tsc.Task.ConfigureAwait(false); - } - } - - /// - /// Returns the current version of the table - /// - /// - public long Version() - { - unsafe - { - return Interop.Methods.table_version(_ptr); - } - } - - /// - /// Returns the current version of the table - /// - /// - public string Uri() - { - unsafe - { - var uri = Interop.Methods.table_uri(_ptr); - try - { - if (uri == null) - { - return string.Empty; - } - - - return ByteArrayRef.StrictUTF8.GetString(uri->data, (int)uri->size); - } - finally - { - Interop.Methods.byte_array_free(_runtime.Ptr, uri); - } - } - } - - public string[] FileUris() - { - unsafe - { - return GetStringArray(Interop.Methods.table_file_uris(_runtime.Ptr, _ptr, null)); - } - } - - public string[] Files() - { - unsafe - { - return GetStringArray(Interop.Methods.table_files(_runtime.Ptr, _ptr, null)); - } - } - - public ProtocolInfo ProtocolVersions() - { - unsafe - { - var response = Methods.table_protocol_versions(_runtime.Ptr, _ptr); - return new ProtocolInfo - { - MinimumReaderVersion = response.min_reader_version, - MinimumWriterVersion = response.min_writer_version, - }; - } - } - - public Schema Schema() - { - unsafe - { - var result = Methods.table_schema(_runtime.Ptr, _ptr); - if (result.error != null) - { - throw DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, result.error); - } - - var schemaPointer = (CArrowSchema*)result.bytes; - try - { - return CArrowSchemaImporter.ImportSchema(schemaPointer); - } - finally - { - CArrowSchema.Free(schemaPointer); - } - } - } - - public async Task InsertAsync( - IReadOnlyCollection records, - Schema schema, - InsertOptions options, - ICancellationToken cancellationToken) - { - if (records.Count == 0) - { - return string.Empty; - } - - using var stream = new RecordBatchReader(records, schema); - return await InsertAsync(stream, options, cancellationToken).ConfigureAwait(false); - } - - public async Task InsertAsync( - IArrowArrayStream stream, - InsertOptions options, - ICancellationToken cancellationToken) - { - var tsc = new TaskCompletionSource(); - using (var scope = new Scope()) - { - unsafe - { - var ffiStream = CArrowArrayStream.Create(); - try - { - CArrowArrayStreamExporter.ExportArrayStream(stream, ffiStream); - Interop.Methods.table_insert( - _runtime.Ptr, - _ptr, - ffiStream, - scope.Pointer(scope.ByteArray(options.Predicate)), - scope.Pointer(ConvertSaveMode(options.SaveMode).Ref), - new UIntPtr(options.MaxRowsPerGroup), - (byte)(options.OverwriteSchema ? 1 : 0), - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((success, fail) => - { - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - return; - } - - if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); - } - else - { - tsc.TrySetResult("{}"); - } - })); - } - finally - { - CArrowArrayStream.Free(ffiStream); - } - } - - return await tsc.Task.ConfigureAwait(false); - } - } - - public async Task MergeAsync( - string query, - IReadOnlyCollection records, - Schema schema, - ICancellationToken cancellationToken) - { - if (records.Count == 0) - { - return string.Empty; - } - - var tsc = new TaskCompletionSource(); - using (var scope = new Scope()) - { - using (var stream = new RecordBatchReader(records, schema)) - { - unsafe - { - var ffiStream = CArrowArrayStream.Create(); - try - { - CArrowArrayStreamExporter.ExportArrayStream(stream, ffiStream); - Interop.Methods.table_merge( - _runtime.Ptr, - _ptr, - scope.Pointer(scope.ByteArray(query)), - ffiStream, - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((success, fail) => - { - - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - } - else if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); - } - else - { - using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); - tsc.TrySetResult(content.ToUTF8()); - } - - })); - } - finally - { - CArrowArrayStream.Free(ffiStream); - } - } - - return await tsc.Task.ConfigureAwait(false); - } - } - } - - public async Task QueryAsync( - string query, - string? tableName, - ICancellationToken cancellationToken) - { - var tsc = new TaskCompletionSource(); - using (var scope = new Scope()) - { - unsafe - { - Methods.table_query( - _runtime.Ptr, - _ptr, - scope.Pointer(scope.ByteArray(query)), - scope.Pointer(scope.ByteArray(tableName)), - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((success, fail) => - { - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - return; - } - - if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); - } - else - { - var stream = CArrowArrayStreamImporter.ImportArrayStream((CArrowArrayStream*)success); - if (!tsc.TrySetResult(stream)) - { - stream.Dispose(); - } - } - })); - } - - return await tsc.Task.ConfigureAwait(false); - } - } - - public async Task DeleteAsync(string predicate, ICancellationToken cancellationToken) - { - var tsc = new TaskCompletionSource(); - using (var scope = new Scope()) - { - unsafe - { - Methods.table_delete( - _runtime.Ptr, - _ptr, - scope.Pointer(scope.ByteArray(predicate)), - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((success, fail) => - { - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - } - else if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); - } - else - { - using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); - tsc.TrySetResult(content.ToUTF8()); - } - })); - - } - - return await tsc.Task.ConfigureAwait(false); - } - } - - public async Task UpdateAsync(string query, ICancellationToken cancellationToken) - { - var tsc = new TaskCompletionSource(); - using (var scope = new Scope()) - { - unsafe - { - Methods.table_update( - _runtime.Ptr, - _ptr, - scope.Pointer(scope.ByteArray(query)), - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((success, fail) => - { - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - } - else if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); - } - else - { - using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); - tsc.TrySetResult(content.ToUTF8()); - } - })); - - } - - return await tsc.Task.ConfigureAwait(false); - } - } - - public async Task HistoryAsync(ulong limit, ICancellationToken cancellationToken) - { - var tsc = new TaskCompletionSource(); - using (var scope = new Scope()) - { - unsafe - { - Methods.history( - _runtime.Ptr, - _ptr, - new UIntPtr(limit), - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((success, fail) => - { - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - } - else if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); - } - else - { - using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); - tsc.TrySetResult(content.ToByteArray()); - } - })); - - } - - return await tsc.Task.ConfigureAwait(false); - } - } - - public async Task AddConstraintAsync(IReadOnlyDictionary constraints, IReadOnlyDictionary? customMetadata, ICancellationToken cancellationToken) - { - if (constraints.Count == 0) - { - return; - } - - var tsc = new TaskCompletionSource(); - using (var scope = new Scope()) - { - unsafe - { - Methods.table_add_constraints( - _runtime.Ptr, - _ptr, - scope.Dictionary(_runtime, constraints), - customMetadata == null ? null : scope.Dictionary(_runtime, customMetadata), - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((fail) => - { - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - } - else if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); - } - else - { - tsc.TrySetResult(true); - } - })); - } - - await tsc.Task.ConfigureAwait(false); - } - } - - public async Task UpdateIncrementalAsync(long? maxVersion, ICancellationToken cancellationToken) - { - var tsc = new TaskCompletionSource(); - using (var scope = new Scope()) - { - unsafe - { - Methods.table_update_incremental( - _runtime.Ptr, - _ptr, - maxVersion ?? -1L, - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((fail) => - { - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - } - else if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); - } - else - { - tsc.TrySetResult(true); - } - })); - - } - - await tsc.Task.ConfigureAwait(false); - } - } - - public DeltaLake.Table.TableMetadata Metadata() - { - unsafe - { - var result = Methods.table_metadata(_runtime.Ptr, _ptr); - if (result.error != null) - { - throw DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, result.error); - } - - try - { - return DeltaLake.Table.TableMetadata.FromUnmanaged(result.metadata); - } - finally - { - var release = (delegate* unmanaged)result.metadata->release; - release(result.metadata); - } - } - } - - - public async Task RestoreAsync(RestoreOptions options, ICancellationToken cancellationToken) - { - var tsc = new TaskCompletionSource(); - using (var scope = new Scope()) - { - unsafe - { - Methods.table_restore( - _runtime.Ptr, - _ptr, - options.Timestamp?.ToUnixTimeMilliseconds() ?? (long?)options.Version ?? 0, - BoolAsByte(options.Timestamp.HasValue), - BoolAsByte(options.IgnoreMissingFiles), - BoolAsByte(options.ProtocolDowngradeAllowed), - scope.Dictionary(_runtime, options.CustomMetadata), - scope.CancellationToken(cancellationToken), - scope.FunctionPointer((fail) => - { - if (cancellationToken.IsCancellationRequested) - { - tsc.TrySetCanceled(cancellationToken); - } - else if (fail != null) - { - tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); - } - else - { - tsc.TrySetResult(true); - } - })); - } - - await tsc.Task.ConfigureAwait(false); - } - } - - internal static ByteArrayRef ConvertSaveMode(SaveMode saveMode) - { - return saveMode switch - { - SaveMode.Append => SaveModeAppend, - SaveMode.Overwrite => SaveModeOverwrite, - SaveMode.ErrorIfExists => SaveModeError, - SaveMode.Ignore => SaveModeIfgnore, - _ => throw new ArgumentOutOfRangeException(nameof(saveMode)), - }; - } - - /// - protected override unsafe bool ReleaseHandle() - { - Interop.Methods.table_free(_ptr); - return true; - } - - private static byte BoolAsByte(bool input) - { - return input switch - { - true => 0, - false => 1, - }; - } - - private unsafe string[] GetStringArray(GenericOrError genericOrError) - { - if (genericOrError.error != null) - { - throw DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, genericOrError.error); - } - - try - { - if (genericOrError.bytes == null) - { - return System.Array.Empty(); - } - - var dynamicArray = (DynamicArray*)genericOrError.bytes; - var uris = new string[(int)dynamicArray->size]; - for (var i = 0; i < uris.Length; i++) - { - var instance = dynamicArray->data + i; - uris[i] = ByteArrayRef.StrictUTF8.GetString(instance->data, (int)instance->size); - } - - return uris; - } - finally - { - Interop.Methods.dynamic_array_free(_runtime.Ptr, (DynamicArray*)genericOrError.bytes); - } - } - } +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Apache.Arrow; +using Apache.Arrow.C; +using Apache.Arrow.Ipc; +using DeltaLake.Bridge.Interop; +using DeltaLake.Errors; +using DeltaLake.Table; +using ICancellationToken = System.Threading.CancellationToken; + +namespace DeltaLake.Bridge +{ + /// + /// Reference to unmanaged delta table + /// + internal sealed class Table : SafeHandle + { + internal static readonly ByteArrayRef SaveModeAppend = ByteArrayRef.FromUTF8("append"); + + internal static readonly ByteArrayRef SaveModeOverwrite = ByteArrayRef.FromUTF8("overwrite"); + + internal static readonly ByteArrayRef SaveModeError = ByteArrayRef.FromUTF8("error"); + + internal static readonly ByteArrayRef SaveModeIfgnore = ByteArrayRef.FromUTF8("ignore"); + + private readonly unsafe Interop.RawDeltaTable* _ptr; + + private readonly Runtime _runtime; + + /// + /// Creates a table + /// + internal unsafe Table(Runtime runtime, Interop.RawDeltaTable* inner) + : base(IntPtr.Zero, true) + { + _ptr = inner; + _runtime = runtime; + SetHandle((IntPtr)_ptr); + } + + /// + public override bool IsInvalid => false; + + /// + /// Returns the current version of the table + /// + /// + public async Task LoadVersionAsync(ulong version, ICancellationToken cancellationToken) + { + var tsc = new TaskCompletionSource(); + using (var scope = new Scope()) + { + unsafe + { + Interop.Methods.table_load_version( + _runtime.Ptr, + _ptr, + unchecked((long)version), + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((fail) => + { + if (cancellationToken.IsCancellationRequested) + { + tsc.TrySetCanceled(cancellationToken); + return; + } + + if (fail != null) + { + tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); + } + else + { + Task.Run(() => tsc.TrySetResult(true)); + } + })); + } + + await tsc.Task.ConfigureAwait(false); + } + } + + internal async Task LoadTimestampAsync(long timestampMilliseconds, ICancellationToken cancellationToken) + { + var tsc = new TaskCompletionSource(); + using (var scope = new Scope()) + { + unsafe + { + Interop.Methods.table_load_with_datetime( + _runtime.Ptr, + _ptr, + timestampMilliseconds, + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((fail) => + { + if (cancellationToken.IsCancellationRequested) + { + tsc.TrySetCanceled(cancellationToken); + return; + } + + if (fail != null) + { + tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); + } + else + { + Task.Run(() => tsc.TrySetResult(true)); + } + })); + } + + await tsc.Task.ConfigureAwait(false); + } + } + + /// + /// Returns the current version of the table + /// + /// + public long Version() + { + unsafe + { + return Interop.Methods.table_version(_ptr); + } + } + + /// + /// Returns the current version of the table + /// + /// + public string Uri() + { + unsafe + { + var uri = Interop.Methods.table_uri(_ptr); + try + { + if (uri == null) + { + return string.Empty; + } + + + return ByteArrayRef.StrictUTF8.GetString(uri->data, (int)uri->size); + } + finally + { + Interop.Methods.byte_array_free(_runtime.Ptr, uri); + } + } + } + + public string[] FileUris() + { + unsafe + { + return GetStringArray(Interop.Methods.table_file_uris(_runtime.Ptr, _ptr, null)); + } + } + + public string[] Files() + { + unsafe + { + return GetStringArray(Interop.Methods.table_files(_runtime.Ptr, _ptr, null)); + } + } + + public ProtocolInfo ProtocolVersions() + { + unsafe + { + var response = Methods.table_protocol_versions(_runtime.Ptr, _ptr); + return new ProtocolInfo + { + MinimumReaderVersion = response.min_reader_version, + MinimumWriterVersion = response.min_writer_version, + }; + } + } + + public Schema Schema() + { + unsafe + { + var result = Methods.table_schema(_runtime.Ptr, _ptr); + if (result.error != null) + { + throw DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, result.error); + } + + var schemaPointer = (CArrowSchema*)result.bytes; + try + { + return CArrowSchemaImporter.ImportSchema(schemaPointer); + } + finally + { + CArrowSchema.Free(schemaPointer); + } + } + } + + public async Task InsertAsync( + IReadOnlyCollection records, + Schema schema, + InsertOptions options, + ICancellationToken cancellationToken) + { + if (records.Count == 0) + { + return string.Empty; + } + + using var stream = new RecordBatchReader(records, schema); + return await InsertAsync(stream, options, cancellationToken).ConfigureAwait(false); + } + + public async Task InsertAsync( + IArrowArrayStream stream, + InsertOptions options, + ICancellationToken cancellationToken) + { + var tsc = new TaskCompletionSource(); + using (var scope = new Scope()) + { + unsafe + { + var ffiStream = CArrowArrayStream.Create(); + try + { + CArrowArrayStreamExporter.ExportArrayStream(stream, ffiStream); + Interop.Methods.table_insert( + _runtime.Ptr, + _ptr, + ffiStream, + scope.Pointer(scope.ByteArray(options.Predicate)), + scope.Pointer(ConvertSaveMode(options.SaveMode).Ref), + new UIntPtr(options.MaxRowsPerGroup), + (byte)(options.OverwriteSchema ? 1 : 0), + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((success, fail) => + { + if (cancellationToken.IsCancellationRequested) + { + tsc.TrySetCanceled(cancellationToken); + return; + } + + if (fail != null) + { + tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); + } + else + { + Task.Run(() => tsc.TrySetResult("{}")); + } + })); + } + finally + { + CArrowArrayStream.Free(ffiStream); + } + } + + return await tsc.Task.ConfigureAwait(false); + } + } + + public async Task MergeAsync( + string query, + IReadOnlyCollection records, + Schema schema, + ICancellationToken cancellationToken) + { + if (records.Count == 0) + { + return string.Empty; + } + + var tsc = new TaskCompletionSource(); + using (var scope = new Scope()) + { + using (var stream = new RecordBatchReader(records, schema)) + { + unsafe + { + var ffiStream = CArrowArrayStream.Create(); + try + { + CArrowArrayStreamExporter.ExportArrayStream(stream, ffiStream); + Interop.Methods.table_merge( + _runtime.Ptr, + _ptr, + scope.Pointer(scope.ByteArray(query)), + ffiStream, + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((success, fail) => + { + + if (cancellationToken.IsCancellationRequested) + { + tsc.TrySetCanceled(cancellationToken); + } + else if (fail != null) + { + tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); + } + else + { + using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); + var value = content.ToUTF8(); + Task.Run(() => tsc.TrySetResult(value)); + } + + })); + } + finally + { + CArrowArrayStream.Free(ffiStream); + } + } + + return await tsc.Task.ConfigureAwait(false); + } + } + } + + public async Task QueryAsync( + string query, + string? tableName, + ICancellationToken cancellationToken) + { + var tsc = new TaskCompletionSource(); + using (var scope = new Scope()) + { + unsafe + { + Methods.table_query( + _runtime.Ptr, + _ptr, + scope.Pointer(scope.ByteArray(query)), + scope.Pointer(scope.ByteArray(tableName)), + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((success, fail) => + { + if (cancellationToken.IsCancellationRequested) + { + tsc.TrySetCanceled(cancellationToken); + return; + } + + if (fail != null) + { + tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); + } + else + { + var stream = CArrowArrayStreamImporter.ImportArrayStream((CArrowArrayStream*)success); + Task.Run(() => { + if (!tsc.TrySetResult(stream)) + { + stream.Dispose(); + } + }); + } + })); + } + + return await tsc.Task.ConfigureAwait(false); + } + } + + public async Task DeleteAsync(string predicate, ICancellationToken cancellationToken) + { + var tsc = new TaskCompletionSource(); + using (var scope = new Scope()) + { + unsafe + { + Methods.table_delete( + _runtime.Ptr, + _ptr, + scope.Pointer(scope.ByteArray(predicate)), + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((success, fail) => + { + if (cancellationToken.IsCancellationRequested) + { + tsc.TrySetCanceled(cancellationToken); + } + else if (fail != null) + { + tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); + } + else + { + using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); + var value = content.ToUTF8(); + Task.Run(() => tsc.TrySetResult(value)); + } + })); + + } + + return await tsc.Task.ConfigureAwait(false); + } + } + + public async Task UpdateAsync(string query, ICancellationToken cancellationToken) + { + var tsc = new TaskCompletionSource(); + using (var scope = new Scope()) + { + unsafe + { + Methods.table_update( + _runtime.Ptr, + _ptr, + scope.Pointer(scope.ByteArray(query)), + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((success, fail) => + { + if (cancellationToken.IsCancellationRequested) + { + tsc.TrySetCanceled(cancellationToken); + } + else if (fail != null) + { + tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); + } + else + { + using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); + var result = content.ToUTF8(); + Task.Run(() => tsc.TrySetResult(result)); + } + })); + + } + + return await tsc.Task.ConfigureAwait(false); + } + } + + public async Task HistoryAsync(ulong limit, ICancellationToken cancellationToken) + { + var tsc = new TaskCompletionSource(); + using (var scope = new Scope()) + { + unsafe + { + Methods.history( + _runtime.Ptr, + _ptr, + new UIntPtr(limit), + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((success, fail) => + { + if (cancellationToken.IsCancellationRequested) + { + tsc.TrySetCanceled(cancellationToken); + } + else if (fail != null) + { + tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); + } + else + { + using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); + var bytes = content.ToByteArray(); + Task.Run(() => tsc.TrySetResult(bytes));; + } + })); + + } + + return await tsc.Task.ConfigureAwait(false); + } + } + + public async Task AddConstraintAsync(IReadOnlyDictionary constraints, IReadOnlyDictionary? customMetadata, ICancellationToken cancellationToken) + { + if (constraints.Count == 0) + { + return; + } + + var tsc = new TaskCompletionSource(); + using (var scope = new Scope()) + { + unsafe + { + Methods.table_add_constraints( + _runtime.Ptr, + _ptr, + scope.Dictionary(_runtime, constraints), + customMetadata == null ? null : scope.Dictionary(_runtime, customMetadata), + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((fail) => + { + if (cancellationToken.IsCancellationRequested) + { + tsc.TrySetCanceled(cancellationToken); + } + else if (fail != null) + { + tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); + } + else + { + Task.Run(() => tsc.TrySetResult(true)); + } + })); + } + + await tsc.Task.ConfigureAwait(false); + } + } + + public async Task UpdateIncrementalAsync(long? maxVersion, ICancellationToken cancellationToken) + { + var tsc = new TaskCompletionSource(); + using (var scope = new Scope()) + { + unsafe + { + Methods.table_update_incremental( + _runtime.Ptr, + _ptr, + maxVersion ?? -1L, + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((fail) => + { + if (cancellationToken.IsCancellationRequested) + { + tsc.TrySetCanceled(cancellationToken); + } + else if (fail != null) + { + tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); + } + else + { + Task.Run(() => tsc.TrySetResult(true)); + } + })); + + } + + await tsc.Task.ConfigureAwait(false); + } + } + + public DeltaLake.Table.TableMetadata Metadata() + { + unsafe + { + var result = Methods.table_metadata(_runtime.Ptr, _ptr); + if (result.error != null) + { + throw DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, result.error); + } + + try + { + return DeltaLake.Table.TableMetadata.FromUnmanaged(result.metadata); + } + finally + { + var release = (delegate* unmanaged)result.metadata->release; + release(result.metadata); + } + } + } + + + public async Task RestoreAsync(RestoreOptions options, ICancellationToken cancellationToken) + { + var tsc = new TaskCompletionSource(); + using (var scope = new Scope()) + { + unsafe + { + Methods.table_restore( + _runtime.Ptr, + _ptr, + options.Timestamp?.ToUnixTimeMilliseconds() ?? (long?)options.Version ?? 0, + BoolAsByte(options.Timestamp.HasValue), + BoolAsByte(options.IgnoreMissingFiles), + BoolAsByte(options.ProtocolDowngradeAllowed), + scope.Dictionary(_runtime, options.CustomMetadata), + scope.CancellationToken(cancellationToken), + scope.FunctionPointer((fail) => + { + if (cancellationToken.IsCancellationRequested) + { + tsc.TrySetCanceled(cancellationToken); + } + else if (fail != null) + { + tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, fail)); + } + else + { + Task.Run(() => tsc.TrySetResult(true)); + } + })); + } + + await tsc.Task.ConfigureAwait(false); + } + } + + internal static ByteArrayRef ConvertSaveMode(SaveMode saveMode) + { + return saveMode switch + { + SaveMode.Append => SaveModeAppend, + SaveMode.Overwrite => SaveModeOverwrite, + SaveMode.ErrorIfExists => SaveModeError, + SaveMode.Ignore => SaveModeIfgnore, + _ => throw new ArgumentOutOfRangeException(nameof(saveMode)), + }; + } + + /// + protected override unsafe bool ReleaseHandle() + { + Interop.Methods.table_free(_ptr); + return true; + } + + private static byte BoolAsByte(bool input) + { + return input switch + { + true => 0, + false => 1, + }; + } + + private unsafe string[] GetStringArray(GenericOrError genericOrError) + { + if (genericOrError.error != null) + { + throw DeltaRuntimeException.FromDeltaTableError(_runtime.Ptr, genericOrError.error); + } + + try + { + if (genericOrError.bytes == null) + { + return System.Array.Empty(); + } + + var dynamicArray = (DynamicArray*)genericOrError.bytes; + var uris = new string[(int)dynamicArray->size]; + for (var i = 0; i < uris.Length; i++) + { + var instance = dynamicArray->data + i; + uris[i] = ByteArrayRef.StrictUTF8.GetString(instance->data, (int)instance->size); + } + + return uris; + } + finally + { + Interop.Methods.dynamic_array_free(_runtime.Ptr, (DynamicArray*)genericOrError.bytes); + } + } + } } \ No newline at end of file diff --git a/src/DeltaLake/Bridge/src/error.rs b/src/DeltaLake/Bridge/src/error.rs index 816072c..c3d77b7 100644 --- a/src/DeltaLake/Bridge/src/error.rs +++ b/src/DeltaLake/Bridge/src/error.rs @@ -1,14 +1,18 @@ +use std::fmt::Debug; + use deltalake::datafusion::sql::sqlparser::parser::ParserError; use crate::{runtime::Runtime, ByteArray}; #[repr(C)] +#[derive(Debug)] pub struct DeltaTableError { code: DeltaTableErrorCode, error: ByteArray, } #[repr(C)] +#[derive(Debug)] pub enum DeltaTableErrorCode { Utf8 = 0, Protocol = 1, diff --git a/src/DeltaLake/Bridge/src/lib.rs b/src/DeltaLake/Bridge/src/lib.rs index 10d800d..e5fce3f 100644 --- a/src/DeltaLake/Bridge/src/lib.rs +++ b/src/DeltaLake/Bridge/src/lib.rs @@ -238,6 +238,7 @@ impl DynamicArray { } #[repr(C)] +#[derive(Debug)] pub struct ByteArray { data: *const u8, size: libc::size_t, diff --git a/src/DeltaLake/Bridge/src/runtime.rs b/src/DeltaLake/Bridge/src/runtime.rs index b218ee4..873b141 100644 --- a/src/DeltaLake/Bridge/src/runtime.rs +++ b/src/DeltaLake/Bridge/src/runtime.rs @@ -117,7 +117,7 @@ pub extern "C" fn dynamic_array_free(runtime: *mut Runtime, array: *const Dynami static HANDLERS: Once = Once::new(); impl Runtime { - fn new(_options: &RuntimeOptions) -> Result { + pub(crate) fn new(_options: &RuntimeOptions) -> Result { HANDLERS.call_once(|| { deltalake::aws::register_handlers(None); deltalake::azure::register_handlers(None); diff --git a/src/DeltaLake/Bridge/src/sql.rs b/src/DeltaLake/Bridge/src/sql.rs index 0630b1a..1b130ea 100644 --- a/src/DeltaLake/Bridge/src/sql.rs +++ b/src/DeltaLake/Bridge/src/sql.rs @@ -346,3 +346,17 @@ impl Iterator for DataFrameStreamIterator { }) } } + +#[cfg(test)] +mod tests { + use crate::runtime::{Runtime, RuntimeOptions}; + + use super::DeltaLakeParser; + + #[test] + fn test_parser() { + let mut runtime = Runtime::new(&RuntimeOptions{}).unwrap(); + let mut parser = DeltaLakeParser::new("UPDATE test SET test = test + CAST(1 AS INT) WHERE test > CAST(1 AS INT)").unwrap(); + parser.parse_update(&mut runtime).unwrap(); + } +} \ No newline at end of file diff --git a/src/DeltaLake/Bridge/src/table.rs b/src/DeltaLake/Bridge/src/table.rs index e18011c..e50792d 100644 --- a/src/DeltaLake/Bridge/src/table.rs +++ b/src/DeltaLake/Bridge/src/table.rs @@ -17,7 +17,7 @@ use deltalake::{ datafusion::{ dataframe::DataFrame, datasource::{MemTable, TableProvider}, - execution::context::{SQLOptions, SessionContext}, sql::sqlparser::ast::AssignmentTarget, + execution::context::{SQLOptions, SessionContext}, sql::sqlparser::ast::{Assignment, AssignmentTarget, Expr}, }, kernel::StructType, operations::{ @@ -920,6 +920,20 @@ pub extern "C" fn table_restore( ); } +fn table_update_internal( + rt: &mut Runtime, + query: NonNull, +) -> Result<(Option, Vec), DeltaTableError> { + let query_str = unsafe { query.as_ref().to_str() }; + let mut parser = DeltaLakeParser::new(query_str) + .map_err(|err| DeltaTableError::new( + rt, + DeltaTableErrorCode::Generic, + &err.to_string(), + ))?; + parser.parse_update(rt) +} + #[no_mangle] pub extern "C" fn table_update( mut runtime: NonNull, @@ -928,21 +942,7 @@ pub extern "C" fn table_update( cancellation_token: Option<&CancellationToken>, callback: GenericErrorCallback, ) { - let query = { - let query = unsafe { query.as_ref() }; - query.to_str() - }; - let mut parser = match DeltaLakeParser::new(query) { - Ok(parser) => parser, - Err(error) => unsafe { - callback( - std::ptr::null(), - DeltaTableError::from_parser_error(runtime.as_mut(), error).into_raw(), - ); - return; - }, - }; - let (predicate, assignments) = match parser.parse_update(unsafe { runtime.as_mut() }) { + let (predicate, assignments) = match table_update_internal(unsafe { runtime.as_mut() }, query) { Ok(statement) => statement, Err(error) => unsafe { callback(std::ptr::null(), error.into_raw()); @@ -971,7 +971,6 @@ pub extern "C" fn table_update( if let Some(predicate) = predicate { ub = ub.with_predicate(predicate.to_string()); } - for assign in assignments { match assign.target { AssignmentTarget::ColumnName(object_name) => { @@ -988,7 +987,7 @@ pub extern "C" fn table_update( }, }; } - + match ub.await { Ok((delta_table, metrics)) => { tbl.table = delta_table; @@ -1001,6 +1000,7 @@ pub extern "C" fn table_update( } } Err(error) => unsafe { + println!("done with error"); callback( std::ptr::null(), DeltaTableError::from_error(rt, error).into_raw(), diff --git a/src/DeltaLake/Table/TableCreateOptions.cs b/src/DeltaLake/Table/TableCreateOptions.cs index b200454..98eab2e 100644 --- a/src/DeltaLake/Table/TableCreateOptions.cs +++ b/src/DeltaLake/Table/TableCreateOptions.cs @@ -1,70 +1,70 @@ -using System.Collections.Generic; -using Apache.Arrow; - -namespace DeltaLake.Table -{ - - /// - /// Options passed when creating a table - /// - public class TableCreateOptions - { - /// - /// Creates an instance of create options, ensuring its mandatory properties - /// - /// Table Uri - /// Table schema in Arrow Format - public TableCreateOptions(string location, Schema schema) - { - Schema = schema; - TableLocation = location; - } - - /// - /// Arrow Schema for the table - /// - public Schema Schema { get; } - - /// - /// Location of the delta table - /// memory://, s3://, azure://, etc - /// - public string TableLocation { get; } - - /// - /// List of columns to use for partitioning - /// - public ICollection PartitionBy { get; init; } = new List(); - - /// - /// Save mode - /// - public SaveMode SaveMode { get; set; } - - /// - /// Optional name for tabe - /// - public string? Name { get; set; } - - /// - /// Optional description for the table - /// - public string? Description { get; set; } - - /// - /// Optional configuration - /// - - public Dictionary? Configuration { get; init; } - - /// - /// Storage options to pass to table builder - /// - public Dictionary? StorageOptions { get; init; } - - /// - /// Table metadata - /// - public Dictionary? CustomMetadata { get; init; } - } +using System.Collections.Generic; +using Apache.Arrow; + +namespace DeltaLake.Table +{ + + /// + /// Options passed when creating a table + /// + public class TableCreateOptions + { + /// + /// Creates an instance of create options, ensuring its mandatory properties + /// + /// Table Uri + /// Table schema in Arrow Format + public TableCreateOptions(string location, Schema schema) + { + Schema = schema; + TableLocation = location; + } + + /// + /// Arrow Schema for the table + /// + public Schema Schema { get; } + + /// + /// Location of the delta table + /// memory://, s3://, azure://, etc + /// + public string TableLocation { get; } + + /// + /// List of columns to use for partitioning + /// + public ICollection PartitionBy { get; init; } = new List(); + + /// + /// Save mode + /// + public SaveMode SaveMode { get; set; } + + /// + /// Optional name for tabe + /// + public string? Name { get; set; } + + /// + /// Optional description for the table + /// + public string? Description { get; set; } + + /// + /// Optional configuration + /// + + public Dictionary? Configuration { get; init; } + + /// + /// Storage options to pass to table builder + /// + public Dictionary? StorageOptions { get; init; } + + /// + /// Table metadata + /// + public Dictionary? CustomMetadata { get; init; } + } } \ No newline at end of file diff --git a/src/DeltaLake/Table/TableMetadata.cs b/src/DeltaLake/Table/TableMetadata.cs index fb5ccdb..a4a913b 100644 --- a/src/DeltaLake/Table/TableMetadata.cs +++ b/src/DeltaLake/Table/TableMetadata.cs @@ -1,113 +1,127 @@ -using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; - -namespace DeltaLake.Table -{ - /// - /// Metadata for the table - /// - public class TableMetadata - { - private static readonly Dictionary EmptySettings = new(); - - /// - /// Table id - /// - public string Id { get; init; } = string.Empty; - - /// - /// Optional table name - /// - public string? Name { get; init; } - - /// - /// Optional table description - /// - public string? Description { get; init; } - - /// - /// Table format provider - /// - public string FormatProvider { get; init; } = string.Empty; - - /// - /// Dictionary of formatting options - /// - public IReadOnlyDictionary FormatOptions { get; init; } = EmptySettings; - - /// - /// Serialized table schema - /// - public string SchemaString { get; init; } = String.Empty; - - /// - /// List of partition columns - /// - public IReadOnlyList PartitionColumns { get; init; } = Array.Empty(); - - /// - /// Created time - /// - public DateTimeOffset CreatedTime { get; init; } - - /// - /// Configuration map - /// - public IReadOnlyDictionary Configuration { get; init; } = EmptySettings; - - internal unsafe static TableMetadata FromUnmanaged(Bridge.Interop.TableMetadata* metadata) - { - var partitionColumns = StringArrayFromPointer(metadata->partition_columns, (int)metadata->partition_columns_count); - return new DeltaLake.Table.TableMetadata - { - Id = Marshal.PtrToStringUTF8(new IntPtr(metadata->id)) ?? string.Empty, - Name = Marshal.PtrToStringUTF8(new IntPtr(metadata->name)), - Description = Marshal.PtrToStringUTF8(new IntPtr(metadata->description)), - FormatProvider = Marshal.PtrToStringUTF8(new IntPtr(metadata->format_provider)) ?? string.Empty, - SchemaString = Marshal.PtrToStringUTF8(new IntPtr(metadata->schema_string)) ?? string.Empty, - CreatedTime = DateTimeOffset.FromUnixTimeMilliseconds(metadata->created_time), - FormatOptions = KeyValueToDictionaryNullable(metadata->format_options), - PartitionColumns = partitionColumns, - Configuration = KeyValueToDictionaryNullable(metadata->configuration), - }; - } - - private unsafe static string[] StringArrayFromPointer(sbyte** pointer, int length) - { - if (pointer == null) - { - return Array.Empty(); - } - - var result = new string[length]; - for (var i = 0; i < length; i++) - { - var entry = *(pointer + i); - result[i] = Marshal.PtrToStringUTF8(new IntPtr(entry)) ?? string.Empty; - } - - return result; - } - - private unsafe static Dictionary KeyValueToDictionaryNullable(Bridge.Interop.Dictionary kvs) - { - var dictionary = new Dictionary(); - if (kvs.length.ToUInt32() <= 0) - { - return dictionary; - } - - for (var i = 0; i < (int)kvs.length; i++) - { - var entry = *(kvs.values + i); - dictionary.TryAdd( - Bridge.ByteArrayRef.StrictUTF8.GetString(entry->key, (int)entry->key_length), - entry->value == null ? null : Bridge.ByteArrayRef.StrictUTF8.GetString(entry->value, (int)entry->value_length) - ); - } - - return dictionary; - } - } +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; + +namespace DeltaLake.Table +{ + /// + /// Metadata for the table + /// + public class TableMetadata + { + private static readonly Dictionary EmptySettings = new(); + + /// + /// Table id + /// + public string Id { get; init; } = string.Empty; + + /// + /// Optional table name + /// + public string? Name { get; init; } + + /// + /// Optional table description + /// + public string? Description { get; init; } + + /// + /// Table format provider + /// + public string FormatProvider { get; init; } = string.Empty; + + /// + /// Dictionary of formatting options + /// + public IReadOnlyDictionary FormatOptions { get; init; } = EmptySettings; + + /// + /// Serialized table schema + /// + public string SchemaString { get; init; } = String.Empty; + + /// + /// List of partition columns + /// + public IReadOnlyList PartitionColumns { get; init; } = Array.Empty(); + + /// + /// Created time + /// + public DateTimeOffset CreatedTime { get; init; } + + /// + /// Configuration map + /// + public IReadOnlyDictionary Configuration { get; init; } = EmptySettings; + + internal unsafe static TableMetadata FromUnmanaged(Bridge.Interop.TableMetadata* metadata) + { + var partitionColumns = StringArrayFromPointer(metadata->partition_columns, (int)metadata->partition_columns_count); + return new DeltaLake.Table.TableMetadata + { + Id = Marshal.PtrToStringUTF8(new IntPtr(metadata->id)) ?? string.Empty, + Name = Marshal.PtrToStringUTF8(new IntPtr(metadata->name)), + Description = Marshal.PtrToStringUTF8(new IntPtr(metadata->description)), + FormatProvider = Marshal.PtrToStringUTF8(new IntPtr(metadata->format_provider)) ?? string.Empty, + SchemaString = Marshal.PtrToStringUTF8(new IntPtr(metadata->schema_string)) ?? string.Empty, + CreatedTime = DateTimeOffset.FromUnixTimeMilliseconds(metadata->created_time), + FormatOptions = KeyValueToDictionaryNullable(metadata->format_options), + PartitionColumns = partitionColumns, + Configuration = KeyValueToDictionary(metadata->configuration), + }; + } + + private unsafe static string[] StringArrayFromPointer(sbyte** pointer, int length) + { + if (pointer == null) + { + return Array.Empty(); + } + + var result = new string[length]; + for (var i = 0; i < length; i++) + { + var entry = *(pointer + i); + result[i] = Marshal.PtrToStringUTF8(new IntPtr(entry)) ?? string.Empty; + } + + return result; + } + + private unsafe static Dictionary KeyValueToDictionaryNullable(Bridge.Interop.Dictionary kvs) + { + var dictionary = new Dictionary(); + if (kvs.length.ToUInt32() <= 0) + { + return dictionary; + } + + for (var i = 0; i < (int)kvs.length; i++) + { + var entry = *(kvs.values + i); + dictionary.TryAdd( + Bridge.ByteArrayRef.StrictUTF8.GetString(entry->key, (int)entry->key_length), + entry->value == null ? null : Bridge.ByteArrayRef.StrictUTF8.GetString(entry->value, (int)entry->value_length) + ); + } + + return dictionary; + } + + private unsafe static Dictionary KeyValueToDictionary(Bridge.Interop.Dictionary kvs) + { + var toReturn = new Dictionary(); + foreach (var (key, value) in KeyValueToDictionaryNullable(kvs).Where(x => x.Value != null)) + { + if (value != null) { + toReturn[key] = value; + } + } + + return toReturn; + } + } } \ No newline at end of file diff --git a/tests/DeltaLake.Tests/DeltaLake.Tests.csproj b/tests/DeltaLake.Tests/DeltaLake.Tests.csproj index 65ec252..d4a0283 100644 --- a/tests/DeltaLake.Tests/DeltaLake.Tests.csproj +++ b/tests/DeltaLake.Tests/DeltaLake.Tests.csproj @@ -6,17 +6,17 @@ false - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - - - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/tests/DeltaLake.Tests/Table/TableHelpers.cs b/tests/DeltaLake.Tests/Table/TableHelpers.cs index d55cab6..1ee1791 100644 --- a/tests/DeltaLake.Tests/Table/TableHelpers.cs +++ b/tests/DeltaLake.Tests/Table/TableHelpers.cs @@ -1,152 +1,153 @@ -using Apache.Arrow; -using Apache.Arrow.Memory; -using Apache.Arrow.Types; -using DeltaLake.Runtime; -using DeltaLake.Table; - -namespace DeltaLake.Tests.Table; - -public enum TableIdentifier -{ - CheckpointWithPartitions, - Checkpoints, - CheckpointsTombstones, - CheckpointsVacuumed, - ConcurrentWorkers, - Covid19NYT, - Delta020, - Delta08Empty, - Delta08, - Delta08Date, - Delta08NullPartition, - Delta08NumericPartition, - Delta08Partition, - Delta08SpecialPartition, - Delta121OnlyStructStats, - Delta220PartitionedTypes, - DeltaLiveTable, - Golden, - HttpRequests, - Issue1374, - SimpleCommit, - SimpleTable, - SimpleTableFeatures, - SimpleTableWithCdc, - SimpleTableWithCheckPoint, - TableWithColumnMapping, - TableWithDeletionLogs, - TableWithEdgeTimestamps, - TableWithLiquidClustering, - TableWithDvSmall, - TableWithoutDvSmall, - WithCheckpointNoLastcheckpoint, -} -public static class TableHelpers -{ - public static readonly IReadOnlyDictionary Tables = new Dictionary - { - [TableIdentifier.CheckpointWithPartitions] = "checkpoint_with_partitions", - [TableIdentifier.Checkpoints] = "checkpoints", - [TableIdentifier.CheckpointsTombstones] = "checkpoints_tombstones", - [TableIdentifier.CheckpointsVacuumed] = "checkpoints_vacuumed", - [TableIdentifier.ConcurrentWorkers] = "concurrent_workers", - [TableIdentifier.Covid19NYT] = "COVID-19_NYT", - [TableIdentifier.Delta020] = "delta-0.2.0", - [TableIdentifier.Delta08Empty] = "delta-0.8-empty", - [TableIdentifier.Delta08] = "delta-0.8.0", - [TableIdentifier.Delta08Date] = "delta-0.8.0-date", - [TableIdentifier.Delta08NullPartition] = "delta-0.8.0-null-partition", - [TableIdentifier.Delta08NumericPartition] = "delta-0.8.0-numeric-partition", - [TableIdentifier.Delta08Partition] = "delta-0.8.0-partitioned", - [TableIdentifier.Delta08SpecialPartition] = "delta-0.8.0-special-partition", - [TableIdentifier.Delta121OnlyStructStats] = "delta-1.2.1-only-struct-stats", - [TableIdentifier.Delta220PartitionedTypes] = "delta-2.2.0-partitioned-types", - [TableIdentifier.DeltaLiveTable] = "delta-live-table", - [TableIdentifier.Golden] = Path.Join("golden", "data-reader-array-primitives"), - [TableIdentifier.HttpRequests] = "http_requests", - [TableIdentifier.Issue1374] = "issue_1374", - [TableIdentifier.SimpleCommit] = "simple_commit", - [TableIdentifier.SimpleTable] = "simple_table", - [TableIdentifier.SimpleTableFeatures] = "simple_table_features", - [TableIdentifier.SimpleTableWithCdc] = "simple_table_with_cdc", - [TableIdentifier.SimpleTableWithCheckPoint] = "simple_table_with_checkpoint", - [TableIdentifier.TableWithColumnMapping] = "table_with_column_mapping", - [TableIdentifier.TableWithDeletionLogs] = "table_with_deletion_logs", - [TableIdentifier.TableWithEdgeTimestamps] = "table_with_edge_timestamps", - [TableIdentifier.TableWithLiquidClustering] = "table_with_liquid_clustering", - [TableIdentifier.TableWithDvSmall] = "table-with-dv-small", - [TableIdentifier.TableWithoutDvSmall] = "table-without-dv-small", - [TableIdentifier.WithCheckpointNoLastcheckpoint] = "with_checkpoint_no_last_checkpoint", - }; - - public static IEnumerable ValidTables => Tables.Keys.Where(t => t switch - { - TableIdentifier.CheckpointsTombstones => false, - _ => true, - }); - - public static string LogPath(this TableIdentifier tid, string? pathRoot = null) - { - return Path.Join(pathRoot ?? Settings.TestRoot, Tables[tid], "_delta_log"); - } - - public static string TablePath(this TableIdentifier tid, string? pathRoot = null) - { - return Path.Join(pathRoot ?? Settings.TestRoot, Tables[tid]); - } - - public static Task<(DeltaRuntime runtime, DeltaTable table)> SetupTable(string path, int length) - { - var options = new InsertOptions - { - SaveMode = SaveMode.Append, - }; - return SetupTable(path, length, options); - } - - public async static Task<(DeltaRuntime runtime, DeltaTable table)> SetupTable( - string path, - int length, - InsertOptions options) - { - var runtime = new DeltaRuntime(RuntimeOptions.Default); - var builder = new Schema.Builder(); - builder.Field(fb => - { - fb.Name("test"); - fb.DataType(Int32Type.Default); - fb.Nullable(false); - }) - .Field(fb => - { - fb.Name("second"); - fb.DataType(StringType.Default); - fb.Nullable(false); - }) - .Field(fb => - { - fb.Name("third"); - fb.DataType(Int64Type.Default); - fb.Nullable(false); - }); - var schema = builder.Build(); - var table = await DeltaTable.CreateAsync( - runtime, - new TableCreateOptions(path, schema), - CancellationToken.None); - Assert.NotNull(table); - - await table.InsertAsync([BuildBasicRecordBatch(length)], schema, options, CancellationToken.None); - return (runtime, table); - } - - public static RecordBatch BuildBasicRecordBatch(int length) - { - var allocator = new NativeMemoryAllocator(); - var recordBatchBuilder = new RecordBatch.Builder(allocator) - .Append("test", false, col => col.Int32(arr => arr.AppendRange(Enumerable.Range(0, length)))) - .Append("second", false, col => col.String(arr => arr.AppendRange(Enumerable.Range(0, length).Select(x => x.ToString())))) - .Append("third", false, col => col.Int64(arr => arr.AppendRange(Enumerable.Range(0, length).Select(x => (long)x)))); - return recordBatchBuilder.Build(); - } -} +using Apache.Arrow; +using Apache.Arrow.Memory; +using Apache.Arrow.Types; +using DeltaLake.Runtime; +using DeltaLake.Table; + +namespace DeltaLake.Tests.Table; + +public enum TableIdentifier +{ + CheckpointWithPartitions, + Checkpoints, + CheckpointsTombstones, + CheckpointsVacuumed, + ConcurrentWorkers, + Covid19NYT, + Delta020, + Delta08Empty, + Delta08, + Delta08Date, + Delta08NullPartition, + Delta08NumericPartition, + Delta08Partition, + Delta08SpecialPartition, + Delta121OnlyStructStats, + Delta220PartitionedTypes, + DeltaLiveTable, + Golden, + HttpRequests, + Issue1374, + SimpleCommit, + SimpleTable, + SimpleTableFeatures, + SimpleTableWithCdc, + SimpleTableWithCheckPoint, + TableWithColumnMapping, + TableWithDeletionLogs, + TableWithEdgeTimestamps, + TableWithLiquidClustering, + TableWithDvSmall, + TableWithoutDvSmall, + WithCheckpointNoLastcheckpoint, +} +public static class TableHelpers +{ + public static readonly IReadOnlyDictionary Tables = new Dictionary + { + [TableIdentifier.CheckpointWithPartitions] = "checkpoint_with_partitions", + [TableIdentifier.Checkpoints] = "checkpoints", + [TableIdentifier.CheckpointsTombstones] = "checkpoints_tombstones", + [TableIdentifier.CheckpointsVacuumed] = "checkpoints_vacuumed", + [TableIdentifier.ConcurrentWorkers] = "concurrent_workers", + [TableIdentifier.Covid19NYT] = "COVID-19_NYT", + [TableIdentifier.Delta020] = "delta-0.2.0", + [TableIdentifier.Delta08Empty] = "delta-0.8-empty", + [TableIdentifier.Delta08] = "delta-0.8.0", + [TableIdentifier.Delta08Date] = "delta-0.8.0-date", + [TableIdentifier.Delta08NullPartition] = "delta-0.8.0-null-partition", + [TableIdentifier.Delta08NumericPartition] = "delta-0.8.0-numeric-partition", + [TableIdentifier.Delta08Partition] = "delta-0.8.0-partitioned", + [TableIdentifier.Delta08SpecialPartition] = "delta-0.8.0-special-partition", + [TableIdentifier.Delta121OnlyStructStats] = "delta-1.2.1-only-struct-stats", + [TableIdentifier.Delta220PartitionedTypes] = "delta-2.2.0-partitioned-types", + [TableIdentifier.DeltaLiveTable] = "delta-live-table", + [TableIdentifier.Golden] = Path.Join("golden", "data-reader-array-primitives"), + [TableIdentifier.HttpRequests] = "http_requests", + [TableIdentifier.Issue1374] = "issue_1374", + [TableIdentifier.SimpleCommit] = "simple_commit", + [TableIdentifier.SimpleTable] = "simple_table", + [TableIdentifier.SimpleTableFeatures] = "simple_table_features", + [TableIdentifier.SimpleTableWithCdc] = "simple_table_with_cdc", + [TableIdentifier.SimpleTableWithCheckPoint] = "simple_table_with_checkpoint", + [TableIdentifier.TableWithColumnMapping] = "table_with_column_mapping", + [TableIdentifier.TableWithDeletionLogs] = "table_with_deletion_logs", + [TableIdentifier.TableWithEdgeTimestamps] = "table_with_edge_timestamps", + [TableIdentifier.TableWithLiquidClustering] = "table_with_liquid_clustering", + [TableIdentifier.TableWithDvSmall] = "table-with-dv-small", + [TableIdentifier.TableWithoutDvSmall] = "table-without-dv-small", + [TableIdentifier.WithCheckpointNoLastcheckpoint] = "with_checkpoint_no_last_checkpoint", + }; + + public static IEnumerable ValidTables => Tables.Keys.Where(t => t switch + { + TableIdentifier.TableWithColumnMapping => false, + TableIdentifier.CheckpointsTombstones => false, + _ => true, + }); + + public static string LogPath(this TableIdentifier tid, string? pathRoot = null) + { + return Path.Join(pathRoot ?? Settings.TestRoot, Tables[tid], "_delta_log"); + } + + public static string TablePath(this TableIdentifier tid, string? pathRoot = null) + { + return Path.Join(pathRoot ?? Settings.TestRoot, Tables[tid]); + } + + public static Task<(DeltaRuntime runtime, DeltaTable table)> SetupTable(string path, int length) + { + var options = new InsertOptions + { + SaveMode = SaveMode.Append, + }; + return SetupTable(path, length, options); + } + + public async static Task<(DeltaRuntime runtime, DeltaTable table)> SetupTable( + string path, + int length, + InsertOptions options) + { + var runtime = new DeltaRuntime(RuntimeOptions.Default); + var builder = new Schema.Builder(); + builder.Field(fb => + { + fb.Name("test"); + fb.DataType(Int32Type.Default); + fb.Nullable(false); + }) + .Field(fb => + { + fb.Name("second"); + fb.DataType(StringType.Default); + fb.Nullable(false); + }) + .Field(fb => + { + fb.Name("third"); + fb.DataType(Int64Type.Default); + fb.Nullable(false); + }); + var schema = builder.Build(); + var table = await DeltaTable.CreateAsync( + runtime, + new TableCreateOptions(path, schema), + CancellationToken.None); + Assert.NotNull(table); + + await table.InsertAsync([BuildBasicRecordBatch(length)], schema, options, CancellationToken.None); + return (runtime, table); + } + + public static RecordBatch BuildBasicRecordBatch(int length) + { + var allocator = new NativeMemoryAllocator(); + var recordBatchBuilder = new RecordBatch.Builder(allocator) + .Append("test", false, col => col.Int32(arr => arr.AppendRange(Enumerable.Range(0, length)))) + .Append("second", false, col => col.String(arr => arr.AppendRange(Enumerable.Range(0, length).Select(x => x.ToString())))) + .Append("third", false, col => col.Int64(arr => arr.AppendRange(Enumerable.Range(0, length).Select(x => (long)x)))); + return recordBatchBuilder.Build(); + } +} diff --git a/tests/DeltaLake.Tests/Table/TableTests.cs b/tests/DeltaLake.Tests/Table/TableTests.cs index d5a2039..954f664 100644 --- a/tests/DeltaLake.Tests/Table/TableTests.cs +++ b/tests/DeltaLake.Tests/Table/TableTests.cs @@ -1,233 +1,230 @@ -using Apache.Arrow; -using Apache.Arrow.Memory; -using Apache.Arrow.Types; -using DeltaLake.Runtime; -using DeltaLake.Table; - -namespace DeltaLake.Tests.Table; - -public class DeltaTableTests -{ - [Fact] - public async Task Create_InMemory_Test() - { - var uri = $"memory://{Guid.NewGuid():N}"; - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - var builder = new Apache.Arrow.Schema.Builder(); - builder.Field(fb => - { - fb.Name("test"); - fb.DataType(Int32Type.Default); - fb.Nullable(false); - }); - var schema = builder.Build(); - using var table = await DeltaTable.CreateAsync( - runtime, - new TableCreateOptions(uri, schema) - { - Configuration = new Dictionary - { - ["delta.dataSkippingNumIndexedCols"] = "32", - ["delta.setTransactionRetentionDuration"] = null, - } - }, - CancellationToken.None); - Assert.NotNull(table); - var version = table.Version(); - Assert.Equal(0UL, version); - var location = table.Location(); - Assert.Equal(uri, location); - var files = table.Files(); - Assert.Empty(files); - var fileUris = table.FileUris(); - Assert.Empty(fileUris); - var returnedSchema = table.Schema(); - Assert.NotNull(returnedSchema); - Assert.Equal(schema.FieldsList.Count, returnedSchema.FieldsList.Count); - var protocol = table.ProtocolVersions(); - Assert.True(protocol.MinimumReaderVersion > 0); - Assert.True(protocol.MinimumWriterVersion > 0); - } - - [Fact] - public async Task Create_Cancellation() - { - await Assert.ThrowsAnyAsync( - async () => - { - var uri = $"memory://{Guid.NewGuid():N}"; - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - var builder = new Apache.Arrow.Schema.Builder(); - builder.Field(fb => - { - fb.Name("test"); - fb.DataType(Int32Type.Default); - fb.Nullable(false); - }); - var schema = builder.Build(); - using var table = await DeltaTable.CreateAsync( - runtime, - new TableCreateOptions(uri, schema) - { - Configuration = new Dictionary - { - ["delta.dataSkippingNumIndexedCols"] = "32", - ["delta.setTransactionRetentionDuration"] = null, - } - }, - new CancellationToken(true)); - }); - } - - [Fact] - public async Task Create_InMemory_With_Partitions_Test() - { - var uri = $"memory://{Guid.NewGuid():N}"; - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - var builder = new Apache.Arrow.Schema.Builder(); - builder.Field(fb => - { - fb.Name("test"); - fb.DataType(Int32Type.Default); - fb.Nullable(false); - }) - .Field(fb => - { - fb.Name("second"); - fb.DataType(Int32Type.Default); - fb.Nullable(false); - }); - var schema = builder.Build(); - var createOptions = new TableCreateOptions(uri, schema) - { - Configuration = new Dictionary - { - ["delta.dataSkippingNumIndexedCols"] = "32", - ["delta.setTransactionRetentionDuration"] = null, - }, - PartitionBy = { "test" }, - Name = "table", - Description = "this table has a description", - CustomMetadata = new Dictionary { ["test"] = "something" }, - StorageOptions = new Dictionary { ["something"] = "here" }, - }; - using var table = await DeltaTable.CreateAsync( - runtime, - createOptions, - CancellationToken.None); - Assert.NotNull(table); - var version = table.Version(); - Assert.Equal(0UL, version); - var location = table.Location(); - Assert.Equal(uri, location); - var metadata = table.Metadata(); - Assert.Single(metadata.PartitionColumns); - Assert.Equal("test", metadata.PartitionColumns[0]); - Assert.Equal(createOptions.Name, metadata.Name); - Assert.Equal(createOptions.Description, metadata.Description); - } - - [Fact] - public async Task Load_Table_Test() - { - var location = Path.Join(Settings.TestRoot, "simple_table"); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), - CancellationToken.None); - Assert.Equal(4UL, table.Version()); - } - - [Fact] - public async Task Load_Cancellation_Test() - { - await Assert.ThrowsAnyAsync(async () => - { - var location = Path.Join(Settings.TestRoot, "simple_table"); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), - new CancellationToken(true)); - Assert.Equal(4UL, table.Version()); - }); - } - - [Fact] - public async Task Load_Table_Memory_Test() - { - var location = Path.Join(Settings.TestRoot, "simple_table"); - var memory = System.Text.Encoding.UTF8.GetBytes(location); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, memory.AsMemory(), new TableOptions(), - CancellationToken.None); - Assert.Equal(4UL, table.Version()); - } - - [Fact] - public async Task Table_Insert_Test() - { - var uri = $"memory://{Guid.NewGuid():N}"; - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - var builder = new Apache.Arrow.Schema.Builder(); - builder.Field(fb => - { - fb.Name("test"); - fb.DataType(Int32Type.Default); - fb.Nullable(false); - }); - var schema = builder.Build(); - using var table = await DeltaTable.CreateAsync( - runtime, - new TableCreateOptions(uri, schema), - CancellationToken.None); - Assert.NotNull(table); - int length = 10; - var allocator = new NativeMemoryAllocator(); - var recordBatchBuilder = new RecordBatch.Builder(allocator) - .Append("test", false, col => col.Int32(arr => arr.AppendRange(Enumerable.Range(0, length)))); - - - var options = new InsertOptions - { - SaveMode = SaveMode.Append, - }; - await table.InsertAsync([recordBatchBuilder.Build()], schema, options, CancellationToken.None); - var version = table.Version(); - var queryResult = table.QueryAsync(new SelectQuery("SELECT test FROM test WHERE test > 1") - { - TableAlias = "test", - }, - CancellationToken.None).ToBlockingEnumerable().ToList(); - Assert.Equal(1UL, version); - var resultCount = 0; - foreach (var batch in queryResult) - { - Assert.Equal(1, batch.ColumnCount); - var column = batch.Column(0); - if (column is not Int32Array integers) - { - throw new Exception("expected int32 array and got " + column.GetType()); - } - - foreach (var intValue in integers) - { - Assert.NotNull(intValue); - Assert.True(intValue.Value > 1); - ++resultCount; - } - } - - Assert.Equal(8, resultCount); - await foreach (var result in table.QueryAsync(new SelectQuery("SELECT test FROM test WHERE test = 1") - { - TableAlias = "test", - }, - CancellationToken.None)) - { - Assert.NotNull(result); - } - var history = await table.HistoryAsync(1, CancellationToken.None); - Assert.Single(history); - await Assert.ThrowsAnyAsync(() => table.HistoryAsync(1, new CancellationToken(true))); - history = await table.HistoryAsync(default, CancellationToken.None); - Assert.Equal(2, history.Length); - } +using Apache.Arrow; +using Apache.Arrow.Memory; +using Apache.Arrow.Types; +using DeltaLake.Runtime; +using DeltaLake.Table; + +namespace DeltaLake.Tests.Table; + +public class DeltaTableTests +{ + [Fact] + public async Task Create_InMemory_Test() + { + var uri = $"memory://{Guid.NewGuid():N}"; + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + var builder = new Apache.Arrow.Schema.Builder(); + builder.Field(fb => + { + fb.Name("test"); + fb.DataType(Int32Type.Default); + fb.Nullable(false); + }); + var schema = builder.Build(); + using var table = await DeltaTable.CreateAsync( + runtime, + new TableCreateOptions(uri, schema) + { + Configuration = new Dictionary + { + ["delta.dataSkippingNumIndexedCols"] = "32", + } + }, + CancellationToken.None); + Assert.NotNull(table); + var version = table.Version(); + Assert.Equal(0UL, version); + var location = table.Location(); + Assert.Equal(uri, location); + var files = table.Files(); + Assert.Empty(files); + var fileUris = table.FileUris(); + Assert.Empty(fileUris); + var returnedSchema = table.Schema(); + Assert.NotNull(returnedSchema); + Assert.Equal(schema.FieldsList.Count, returnedSchema.FieldsList.Count); + var protocol = table.ProtocolVersions(); + Assert.True(protocol.MinimumReaderVersion > 0); + Assert.True(protocol.MinimumWriterVersion > 0); + } + + [Fact] + public async Task Create_Cancellation() + { + await Assert.ThrowsAnyAsync( + async () => + { + var uri = $"memory://{Guid.NewGuid():N}"; + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + var builder = new Apache.Arrow.Schema.Builder(); + builder.Field(fb => + { + fb.Name("test"); + fb.DataType(Int32Type.Default); + fb.Nullable(false); + }); + var schema = builder.Build(); + using var table = await DeltaTable.CreateAsync( + runtime, + new TableCreateOptions(uri, schema) + { + Configuration = new Dictionary + { + ["delta.dataSkippingNumIndexedCols"] = "32", + } + }, + new CancellationToken(true)); + }); + } + + [Fact] + public async Task Create_InMemory_With_Partitions_Test() + { + var uri = $"memory://{Guid.NewGuid():N}"; + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + var builder = new Apache.Arrow.Schema.Builder(); + builder.Field(fb => + { + fb.Name("test"); + fb.DataType(Int32Type.Default); + fb.Nullable(false); + }) + .Field(fb => + { + fb.Name("second"); + fb.DataType(Int32Type.Default); + fb.Nullable(false); + }); + var schema = builder.Build(); + var createOptions = new TableCreateOptions(uri, schema) + { + Configuration = new Dictionary + { + ["delta.dataSkippingNumIndexedCols"] = "32", + }, + PartitionBy = { "test" }, + Name = "table", + Description = "this table has a description", + CustomMetadata = new Dictionary { ["test"] = "something" }, + StorageOptions = new Dictionary { ["something"] = "here" }, + }; + using var table = await DeltaTable.CreateAsync( + runtime, + createOptions, + CancellationToken.None); + Assert.NotNull(table); + var version = table.Version(); + Assert.Equal(0UL, version); + var location = table.Location(); + Assert.Equal(uri, location); + var metadata = table.Metadata(); + Assert.Single(metadata.PartitionColumns); + Assert.Equal("test", metadata.PartitionColumns[0]); + Assert.Equal(createOptions.Name, metadata.Name); + Assert.Equal(createOptions.Description, metadata.Description); + } + + [Fact] + public async Task Load_Table_Test() + { + var location = Path.Join(Settings.TestRoot, "simple_table"); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), + CancellationToken.None); + Assert.Equal(4UL, table.Version()); + } + + [Fact] + public async Task Load_Cancellation_Test() + { + await Assert.ThrowsAnyAsync(async () => + { + var location = Path.Join(Settings.TestRoot, "simple_table"); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), + new CancellationToken(true)); + Assert.Equal(4UL, table.Version()); + }); + } + + [Fact] + public async Task Load_Table_Memory_Test() + { + var location = Path.Join(Settings.TestRoot, "simple_table"); + var memory = System.Text.Encoding.UTF8.GetBytes(location); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, memory.AsMemory(), new TableOptions(), + CancellationToken.None); + Assert.Equal(4UL, table.Version()); + } + + [Fact] + public async Task Table_Insert_Test() + { + var uri = $"memory://{Guid.NewGuid():N}"; + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + var builder = new Apache.Arrow.Schema.Builder(); + builder.Field(fb => + { + fb.Name("test"); + fb.DataType(Int32Type.Default); + fb.Nullable(false); + }); + var schema = builder.Build(); + using var table = await DeltaTable.CreateAsync( + runtime, + new TableCreateOptions(uri, schema), + CancellationToken.None); + Assert.NotNull(table); + int length = 10; + var allocator = new NativeMemoryAllocator(); + var recordBatchBuilder = new RecordBatch.Builder(allocator) + .Append("test", false, col => col.Int32(arr => arr.AppendRange(Enumerable.Range(0, length)))); + + + var options = new InsertOptions + { + SaveMode = SaveMode.Append, + }; + await table.InsertAsync([recordBatchBuilder.Build()], schema, options, CancellationToken.None); + var version = table.Version(); + var queryResult = table.QueryAsync(new SelectQuery("SELECT test FROM test WHERE test > 1") + { + TableAlias = "test", + }, + CancellationToken.None).ToBlockingEnumerable().ToList(); + Assert.Equal(1UL, version); + var resultCount = 0; + foreach (var batch in queryResult) + { + Assert.Equal(1, batch.ColumnCount); + var column = batch.Column(0); + if (column is not Int32Array integers) + { + throw new Exception("expected int32 array and got " + column.GetType()); + } + + foreach (var intValue in integers) + { + Assert.NotNull(intValue); + Assert.True(intValue.Value > 1); + ++resultCount; + } + } + + Assert.Equal(8, resultCount); + await foreach (var result in table.QueryAsync(new SelectQuery("SELECT test FROM test WHERE test = 1") + { + TableAlias = "test", + }, + CancellationToken.None)) + { + Assert.NotNull(result); + } + var history = await table.HistoryAsync(1, CancellationToken.None); + Assert.Single(history); + await Assert.ThrowsAnyAsync(() => table.HistoryAsync(1, new CancellationToken(true))); + history = await table.HistoryAsync(default, CancellationToken.None); + Assert.Equal(2, history.Length); + } } \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Table/UpdateTests.cs b/tests/DeltaLake.Tests/Table/UpdateTests.cs index d347457..a4f4034 100644 --- a/tests/DeltaLake.Tests/Table/UpdateTests.cs +++ b/tests/DeltaLake.Tests/Table/UpdateTests.cs @@ -1,97 +1,97 @@ -using Apache.Arrow; -using DeltaLake.Errors; -using DeltaLake.Table; - -namespace DeltaLake.Tests.Table; -public class UpdateTests -{ - private const string UpdateGreaterThanOne = "UPDATE test SET test = test + CAST(1 AS INT) WHERE test > CAST(1 AS INT)"; - private const string UpdateNoPredicate = "UPDATE test SET test = test + CAST(1 AS INT)"; - - private const string UpdatePredicateMiss = "UPDATE test SET test = test + CAST(1 AS INT) WHERE test < CAST(0 as INT)"; - - [Theory] - [InlineData(1, UpdateGreaterThanOne)] - [InlineData(2, UpdateGreaterThanOne)] - [InlineData(10, UpdateGreaterThanOne)] - [InlineData(100, UpdateGreaterThanOne)] - public async Task Memory_Update_Variable_Record_Test( - int length, - string predicate) - { - var totalValue = length < 2 ? length - 1 : (length - 2) / 2 * (length + 3) + 1; - await BaseUpdateTest($"memory://{Guid.NewGuid():N}", length, predicate, totalValue); - } - - [Theory] - [InlineData(1, UpdateNoPredicate)] - [InlineData(2, UpdateNoPredicate)] - [InlineData(10, UpdateNoPredicate)] - [InlineData(100, UpdateNoPredicate)] - public async Task Memory_Update_No_Predicate_Variable_Record_Test( - int length, - string predicate) - { - var totalValue = length == 1 ? 1 : length / 2 * (length + 1); - await BaseUpdateTest($"memory://{Guid.NewGuid():N}", length, predicate, totalValue); - } - - [Theory] - [InlineData(1, UpdatePredicateMiss)] - [InlineData(2, UpdatePredicateMiss)] - [InlineData(10, UpdatePredicateMiss)] - [InlineData(100, UpdatePredicateMiss)] - public async Task Memory_Update_Predicate_Miss_Variable_Record_Test( - int length, - string predicate) - { - var totalValue = length / 2 * (length - 1); - await BaseUpdateTest($"memory://{Guid.NewGuid():N}", length, predicate, totalValue); - } - - [Fact] - public async Task Memory_Update_Invalid_Predicate__Test() - { - await Assert.ThrowsAsync(() => - BaseUpdateTest($"memory://{Guid.NewGuid():N}", 10, "predicate", 0)); - } - - [Fact] - public async Task Update_Cancellation_Test() - { - var data = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 10); - using var runtime = data.runtime; - using var table = data.table; - var version = table.Version(); - try - { - await table.UpdateAsync("this will throw", new CancellationToken(true)); - throw new InvalidOperationException(); - } - catch (OperationCanceledException) - { - Assert.Equal(version, table.Version()); - } - } - - private async static Task BaseUpdateTest( - string path, - int length, - string query, - long expectedTotal) - { - var data = await TableHelpers.SetupTable(path, length); - using var runtime = data.runtime; - using var table = data.table; - await table.UpdateAsync(query, CancellationToken.None); - var queryResult = table.QueryAsync(new SelectQuery("select SUM(test) from test") - { - TableAlias = "test", - }, - CancellationToken.None).ToBlockingEnumerable().ToList(); - var totalValue = queryResult.Select(rb => ((Int64Array)rb.Column(0)).Sum(i => i!.Value)).Sum(); - var totalRecords = queryResult.Select(s => s.Length).Sum(); - Assert.Equal(1, totalRecords); - Assert.Equal(expectedTotal, totalValue); - } +using Apache.Arrow; +using DeltaLake.Errors; +using DeltaLake.Table; + +namespace DeltaLake.Tests.Table; +public class UpdateTests +{ + private const string UpdateGreaterThanOne = "UPDATE test SET test = test + CAST(1 AS INT) WHERE test > CAST(1 AS INT)"; + private const string UpdateNoPredicate = "UPDATE test SET test = test + CAST(1 AS INT)"; + + private const string UpdatePredicateMiss = "UPDATE test SET test = test + CAST(1 AS INT) WHERE test < CAST(0 as INT)"; + + [Theory] + [InlineData(1, UpdateGreaterThanOne)] + [InlineData(2, UpdateGreaterThanOne)] + [InlineData(10, UpdateGreaterThanOne)] + [InlineData(100, UpdateGreaterThanOne)] + public async Task Memory_Update_Variable_Record_Test( + int length, + string predicate) + { + var totalValue = length < 2 ? length - 1 : (length - 2) / 2 * (length + 3) + 1; + await BaseUpdateTest($"memory://{Guid.NewGuid():N}", length, predicate, totalValue); + } + + [Theory] + [InlineData(1, UpdateNoPredicate)] + [InlineData(2, UpdateNoPredicate)] + [InlineData(10, UpdateNoPredicate)] + [InlineData(100, UpdateNoPredicate)] + public async Task Memory_Update_No_Predicate_Variable_Record_Test( + int length, + string predicate) + { + var totalValue = length == 1 ? 1 : length / 2 * (length + 1); + await BaseUpdateTest($"memory://{Guid.NewGuid():N}", length, predicate, totalValue); + } + + [Theory] + [InlineData(1, UpdatePredicateMiss)] + [InlineData(2, UpdatePredicateMiss)] + [InlineData(10, UpdatePredicateMiss)] + [InlineData(100, UpdatePredicateMiss)] + public async Task Memory_Update_Predicate_Miss_Variable_Record_Test( + int length, + string predicate) + { + var totalValue = length / 2 * (length - 1); + await BaseUpdateTest($"memory://{Guid.NewGuid():N}", length, predicate, totalValue); + } + + [Fact] + public async Task Memory_Update_Invalid_Predicate__Test() + { + await Assert.ThrowsAsync(() => + BaseUpdateTest($"memory://{Guid.NewGuid():N}", 10, "predicate", 0)); + } + + [Fact] + public async Task Update_Cancellation_Test() + { + var data = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 10); + using var runtime = data.runtime; + using var table = data.table; + var version = table.Version(); + try + { + await table.UpdateAsync("this will throw", new CancellationToken(true)); + throw new InvalidOperationException(); + } + catch (OperationCanceledException) + { + Assert.Equal(version, table.Version()); + } + } + + private async static Task BaseUpdateTest( + string path, + int length, + string query, + long expectedTotal) + { + var data = await TableHelpers.SetupTable(path, length); + using var runtime = data.runtime; + using var table = data.table; + await table.UpdateAsync(query, CancellationToken.None); + var queryResult = table.QueryAsync(new SelectQuery("select SUM(test) from test") + { + TableAlias = "test", + }, + CancellationToken.None).ToBlockingEnumerable().ToList(); + var totalValue = queryResult.Select(rb => ((Int64Array)rb.Column(0)).Sum(i => i!.Value)).Sum(); + var totalRecords = queryResult.Select(s => s.Length).Sum(); + Assert.Equal(1, totalRecords); + Assert.Equal(expectedTotal, totalValue); + } } \ No newline at end of file From 2e6fc057a569497207412ff00efbde2aed3e0209 Mon Sep 17 00:00:00 2001 From: KyJah Keys Date: Fri, 27 Sep 2024 14:44:04 -0400 Subject: [PATCH 06/12] dotnet format --- src/DeltaLake/Bridge/ByteArrayRef.cs | 236 +++--- src/DeltaLake/Bridge/CancellationToken.cs | 170 ++--- src/DeltaLake/Bridge/Map.cs | 182 ++--- src/DeltaLake/Bridge/RecordBatchReader.cs | 72 +- src/DeltaLake/Bridge/RentedByteArrayRef.cs | 70 +- src/DeltaLake/Bridge/Runtime.cs | 4 +- src/DeltaLake/Bridge/Scope.cs | 342 ++++----- src/DeltaLake/Bridge/Table.cs | 11 +- .../Errors/DeltaConfigurationException.cs | 60 +- src/DeltaLake/Errors/DeltaLakeException.cs | 76 +- src/DeltaLake/Errors/DeltaRuntimeException.cs | 116 +-- src/DeltaLake/Runtime/DeltaRuntime.cs | 78 +- src/DeltaLake/Runtime/RuntimeOptions.cs | 26 +- src/DeltaLake/Table/CommitInfo.cs | 138 ++-- src/DeltaLake/Table/DeltaTable.cs | 712 +++++++++--------- src/DeltaLake/Table/InsertOptions.cs | 66 +- src/DeltaLake/Table/ProtocolInfo.cs | 38 +- src/DeltaLake/Table/RestoreOptions.cs | 70 +- src/DeltaLake/Table/SaveMode.cs | 48 +- src/DeltaLake/Table/SelectQuery.cs | 58 +- src/DeltaLake/Table/TableMetadata.cs | 3 +- src/DeltaLake/Table/TableOptions.cs | 58 +- tests/DeltaLake.Tests/Runtime/RuntimeTests.cs | 24 +- tests/DeltaLake.Tests/Settings.cs | 12 +- .../Table/AllTablesEnumerable.cs | 26 +- .../DeltaLake.Tests/Table/ConstraintTests.cs | 328 ++++---- tests/DeltaLake.Tests/Table/DeleteTests.cs | 226 +++--- tests/DeltaLake.Tests/Table/InsertTests.cs | 282 +++---- tests/DeltaLake.Tests/Table/LoadTests.cs | 644 ++++++++-------- tests/DeltaLake.Tests/Table/MergeTests.cs | 436 +++++------ tests/DeltaLake.Tests/Table/QueryTests.cs | 130 ++-- tests/DeltaLake.Tests/Table/RestoreTests.cs | 66 +- tests/DeltaLake.Tests/Usings.cs | 2 +- 33 files changed, 2406 insertions(+), 2404 deletions(-) diff --git a/src/DeltaLake/Bridge/ByteArrayRef.cs b/src/DeltaLake/Bridge/ByteArrayRef.cs index 87f5458..3a2854f 100644 --- a/src/DeltaLake/Bridge/ByteArrayRef.cs +++ b/src/DeltaLake/Bridge/ByteArrayRef.cs @@ -1,119 +1,119 @@ -using System; -using System.Buffers; -using System.Text; - -namespace DeltaLake.Bridge -{ - /// - /// Representation of a byte array owned by .NET. Users should usually use a - /// - internal sealed class ByteArrayRef : IDisposable - { - private readonly MemoryHandle bytesHandle; - - /// - /// Initializes a new instance of the class. - /// - /// Byte array to use. - public ByteArrayRef(byte[] bytes) - : this(bytes, bytes.Length) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// Byte array to use. - /// Amount of bytes to use. - public ByteArrayRef(byte[] bytes, int length) - : this(new Memory(bytes, 0, length)) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// Byte array to use. - public ByteArrayRef(Memory bytes) - { - Bytes = bytes; - bytesHandle = bytes.Pin(); - unsafe - { - Ref = new Interop.ByteArrayRef() - { - data = (byte*)bytesHandle.Pointer, - size = (UIntPtr)bytes.Length, - }; - } - } - - /// - public void Dispose() - { - bytesHandle.Dispose(); - GC.SuppressFinalize(this); - } - - /// - /// Gets empty byte array. - /// - public static ByteArrayRef Empty { get; } = new(Array.Empty()); - - /// - /// Gets current byte array for this ref. - /// - public Memory Bytes { get; private init; } - - /// - /// Gets internal ref. - /// - public Interop.ByteArrayRef Ref { get; private init; } - - /// - /// Gets strict UTF-8 encoding. - /// - internal static UTF8Encoding StrictUTF8 { get; } = new(false, true); - - /// - /// Convert a string to a UTF-8 byte array. - /// - /// String to convert. - /// Converted byte array. - public static ByteArrayRef FromUTF8(string s) - { - if (s.Length == 0) - { - return Empty; - } - - return new ByteArrayRef(StrictUTF8.GetBytes(s)); - } - - /// - /// Convert a string to a UTF-8 byte array. - /// - /// String to convert. - /// Converted byte array. - public static RentedByteArrayRef RentUtf8(string s) - { - if (s.Length == 0) - { - - return RentedByteArrayRef.Empty; - } - - var bytes = ArrayPool.Shared.Rent(StrictUTF8.GetByteCount(s)); - var length = StrictUTF8.GetBytes(s, bytes); - return new RentedByteArrayRef(new ByteArrayRef(bytes.AsMemory(0, length)), bytes, ArrayPool.Shared); - } - - /// - /// Copy a byte array ref contents to a UTF8 string. - /// - /// Byte array ref. - /// String. - public static unsafe string ToUtf8(Interop.ByteArrayRef byteArray) => - StrictUTF8.GetString(byteArray.data, (int)byteArray.size); - } +using System; +using System.Buffers; +using System.Text; + +namespace DeltaLake.Bridge +{ + /// + /// Representation of a byte array owned by .NET. Users should usually use a + /// + internal sealed class ByteArrayRef : IDisposable + { + private readonly MemoryHandle bytesHandle; + + /// + /// Initializes a new instance of the class. + /// + /// Byte array to use. + public ByteArrayRef(byte[] bytes) + : this(bytes, bytes.Length) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Byte array to use. + /// Amount of bytes to use. + public ByteArrayRef(byte[] bytes, int length) + : this(new Memory(bytes, 0, length)) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Byte array to use. + public ByteArrayRef(Memory bytes) + { + Bytes = bytes; + bytesHandle = bytes.Pin(); + unsafe + { + Ref = new Interop.ByteArrayRef() + { + data = (byte*)bytesHandle.Pointer, + size = (UIntPtr)bytes.Length, + }; + } + } + + /// + public void Dispose() + { + bytesHandle.Dispose(); + GC.SuppressFinalize(this); + } + + /// + /// Gets empty byte array. + /// + public static ByteArrayRef Empty { get; } = new(Array.Empty()); + + /// + /// Gets current byte array for this ref. + /// + public Memory Bytes { get; private init; } + + /// + /// Gets internal ref. + /// + public Interop.ByteArrayRef Ref { get; private init; } + + /// + /// Gets strict UTF-8 encoding. + /// + internal static UTF8Encoding StrictUTF8 { get; } = new(false, true); + + /// + /// Convert a string to a UTF-8 byte array. + /// + /// String to convert. + /// Converted byte array. + public static ByteArrayRef FromUTF8(string s) + { + if (s.Length == 0) + { + return Empty; + } + + return new ByteArrayRef(StrictUTF8.GetBytes(s)); + } + + /// + /// Convert a string to a UTF-8 byte array. + /// + /// String to convert. + /// Converted byte array. + public static RentedByteArrayRef RentUtf8(string s) + { + if (s.Length == 0) + { + + return RentedByteArrayRef.Empty; + } + + var bytes = ArrayPool.Shared.Rent(StrictUTF8.GetByteCount(s)); + var length = StrictUTF8.GetBytes(s, bytes); + return new RentedByteArrayRef(new ByteArrayRef(bytes.AsMemory(0, length)), bytes, ArrayPool.Shared); + } + + /// + /// Copy a byte array ref contents to a UTF8 string. + /// + /// Byte array ref. + /// String. + public static unsafe string ToUtf8(Interop.ByteArrayRef byteArray) => + StrictUTF8.GetString(byteArray.data, (int)byteArray.size); + } } \ No newline at end of file diff --git a/src/DeltaLake/Bridge/CancellationToken.cs b/src/DeltaLake/Bridge/CancellationToken.cs index beab4a2..a4c5f79 100644 --- a/src/DeltaLake/Bridge/CancellationToken.cs +++ b/src/DeltaLake/Bridge/CancellationToken.cs @@ -1,86 +1,86 @@ -using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; - -namespace DeltaLake.Bridge -{ - /// - /// Core-owned cancellation token. - /// - internal sealed class CancellationToken : SafeHandle - { - private readonly List _cancellationRegistrations = new List(); - - /// - /// Initializes a new instance of the class. - /// - public CancellationToken() - : base(IntPtr.Zero, true) - { - unsafe - { - Ptr = Interop.Methods.cancellation_token_new(); - SetHandle((IntPtr)Ptr); - } - } - - /// - public override unsafe bool IsInvalid => false; - - /// - /// Gets internal token pointer. - /// - internal unsafe Interop.CancellationToken* Ptr { get; private init; } - - /// - /// Create a core cancellation token from the given cancellation token. - /// - /// Threading token. - /// Created cancellation token. - public static CancellationToken FromThreading(System.Threading.CancellationToken token) - { - var ret = new CancellationToken(); - if (token.IsCancellationRequested) - { - ret.Cancel(); - } - else - { - ret._cancellationRegistrations.Add(token.Register(ret.Cancel)); - } - - return ret; - } - - /// - /// Cancel this token. - /// - public void Cancel() - { - unsafe - { - Interop.Methods.cancellation_token_cancel(Ptr); - } - } - - /// - protected override unsafe bool ReleaseHandle() - { - Interop.Methods.cancellation_token_free(Ptr); - return true; - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - - if (disposing) - { - foreach (var registration in _cancellationRegistrations) - { - registration.Dispose(); - } - } - } - } +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace DeltaLake.Bridge +{ + /// + /// Core-owned cancellation token. + /// + internal sealed class CancellationToken : SafeHandle + { + private readonly List _cancellationRegistrations = new List(); + + /// + /// Initializes a new instance of the class. + /// + public CancellationToken() + : base(IntPtr.Zero, true) + { + unsafe + { + Ptr = Interop.Methods.cancellation_token_new(); + SetHandle((IntPtr)Ptr); + } + } + + /// + public override unsafe bool IsInvalid => false; + + /// + /// Gets internal token pointer. + /// + internal unsafe Interop.CancellationToken* Ptr { get; private init; } + + /// + /// Create a core cancellation token from the given cancellation token. + /// + /// Threading token. + /// Created cancellation token. + public static CancellationToken FromThreading(System.Threading.CancellationToken token) + { + var ret = new CancellationToken(); + if (token.IsCancellationRequested) + { + ret.Cancel(); + } + else + { + ret._cancellationRegistrations.Add(token.Register(ret.Cancel)); + } + + return ret; + } + + /// + /// Cancel this token. + /// + public void Cancel() + { + unsafe + { + Interop.Methods.cancellation_token_cancel(Ptr); + } + } + + /// + protected override unsafe bool ReleaseHandle() + { + Interop.Methods.cancellation_token_free(Ptr); + return true; + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + if (disposing) + { + foreach (var registration in _cancellationRegistrations) + { + registration.Dispose(); + } + } + } + } } \ No newline at end of file diff --git a/src/DeltaLake/Bridge/Map.cs b/src/DeltaLake/Bridge/Map.cs index 0d910a4..0ce3c57 100644 --- a/src/DeltaLake/Bridge/Map.cs +++ b/src/DeltaLake/Bridge/Map.cs @@ -1,92 +1,92 @@ -using System.Collections.Generic; -using System.Runtime.InteropServices; - -namespace DeltaLake.Bridge -{ - internal sealed class Map - { - private unsafe Map(Interop.Map* inner) - { - Ref = inner; - } - - public unsafe Interop.Map* Ref { get; } - - public static unsafe Map FromDictionary(Runtime runtime, IReadOnlyCollection> source) - { - var map = Interop.Methods.map_new(runtime.Ptr, (nuint)source.Count); - foreach (var (key, value) in source) - { - using var keyRef = ByteArrayRef.RentUtf8(key); - using var valueRef = ByteArrayRef.RentUtf8(value); - WriteTuple(map, keyRef.Ref, valueRef.Ref); - } - - return new Map(map); - } - - public static unsafe Map FromOptionalDictionary(Runtime runtime, IReadOnlyCollection> source) - { - var map = Interop.Methods.map_new(runtime.Ptr, (nuint)source.Count); - foreach (var (key, value) in source) - { - using var keyRef = ByteArrayRef.RentUtf8(key); - if (value != null) - { - using var valueRef = ByteArrayRef.RentUtf8(value); - WriteTuple(map, keyRef.Ref, valueRef.Ref); - } - else - { - WriteSingle(map, keyRef.Ref); - } - - } - - return new Map(map); - } - - private static unsafe void WriteTuple(Interop.Map* map, ByteArrayRef keyRef, ByteArrayRef valueRef) - { - - var keyRefHandle = GCHandle.Alloc(keyRef.Ref, GCHandleType.Pinned); - try - { - var valueRefHandle = GCHandle.Alloc(valueRef.Ref, GCHandleType.Pinned); - try - { - Interop.Methods.map_add( - map, - (Interop.ByteArrayRef*)keyRefHandle.AddrOfPinnedObject(), - (Interop.ByteArrayRef*)valueRefHandle.AddrOfPinnedObject()); - } - finally - { - valueRefHandle.Free(); - } - } - finally - { - keyRefHandle.Free(); - } - } - - private static unsafe void WriteSingle(Interop.Map* map, ByteArrayRef keyRef) - { - - var keyRefHandle = GCHandle.Alloc(keyRef.Ref, GCHandleType.Pinned); - try - { - Interop.Methods.map_add( - map, - (Interop.ByteArrayRef*)keyRefHandle.AddrOfPinnedObject(), - null); - - } - finally - { - keyRefHandle.Free(); - } - } - } +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace DeltaLake.Bridge +{ + internal sealed class Map + { + private unsafe Map(Interop.Map* inner) + { + Ref = inner; + } + + public unsafe Interop.Map* Ref { get; } + + public static unsafe Map FromDictionary(Runtime runtime, IReadOnlyCollection> source) + { + var map = Interop.Methods.map_new(runtime.Ptr, (nuint)source.Count); + foreach (var (key, value) in source) + { + using var keyRef = ByteArrayRef.RentUtf8(key); + using var valueRef = ByteArrayRef.RentUtf8(value); + WriteTuple(map, keyRef.Ref, valueRef.Ref); + } + + return new Map(map); + } + + public static unsafe Map FromOptionalDictionary(Runtime runtime, IReadOnlyCollection> source) + { + var map = Interop.Methods.map_new(runtime.Ptr, (nuint)source.Count); + foreach (var (key, value) in source) + { + using var keyRef = ByteArrayRef.RentUtf8(key); + if (value != null) + { + using var valueRef = ByteArrayRef.RentUtf8(value); + WriteTuple(map, keyRef.Ref, valueRef.Ref); + } + else + { + WriteSingle(map, keyRef.Ref); + } + + } + + return new Map(map); + } + + private static unsafe void WriteTuple(Interop.Map* map, ByteArrayRef keyRef, ByteArrayRef valueRef) + { + + var keyRefHandle = GCHandle.Alloc(keyRef.Ref, GCHandleType.Pinned); + try + { + var valueRefHandle = GCHandle.Alloc(valueRef.Ref, GCHandleType.Pinned); + try + { + Interop.Methods.map_add( + map, + (Interop.ByteArrayRef*)keyRefHandle.AddrOfPinnedObject(), + (Interop.ByteArrayRef*)valueRefHandle.AddrOfPinnedObject()); + } + finally + { + valueRefHandle.Free(); + } + } + finally + { + keyRefHandle.Free(); + } + } + + private static unsafe void WriteSingle(Interop.Map* map, ByteArrayRef keyRef) + { + + var keyRefHandle = GCHandle.Alloc(keyRef.Ref, GCHandleType.Pinned); + try + { + Interop.Methods.map_add( + map, + (Interop.ByteArrayRef*)keyRefHandle.AddrOfPinnedObject(), + null); + + } + finally + { + keyRefHandle.Free(); + } + } + } } \ No newline at end of file diff --git a/src/DeltaLake/Bridge/RecordBatchReader.cs b/src/DeltaLake/Bridge/RecordBatchReader.cs index 895ee06..4f4f334 100644 --- a/src/DeltaLake/Bridge/RecordBatchReader.cs +++ b/src/DeltaLake/Bridge/RecordBatchReader.cs @@ -1,37 +1,37 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using Apache.Arrow; -using Apache.Arrow.Ipc; - -namespace DeltaLake.Bridge -{ - internal sealed class RecordBatchReader : IArrowArrayStream - { - private readonly IEnumerator _enumerator; - - public RecordBatchReader(IEnumerable recordBatches, Schema schema) - { - _enumerator = recordBatches.GetEnumerator(); - Schema = schema; - } - - public Schema Schema { get; } - - public void Dispose() - { - _enumerator.Dispose(); - } - - public ValueTask ReadNextRecordBatchAsync(System.Threading.CancellationToken cancellationToken = default) - { - if (_enumerator.MoveNext()) - { - return ValueTask.FromResult(_enumerator.Current); - } - -#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. - return ValueTask.FromResult(default); -#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. - } - } +using System.Collections.Generic; +using System.Threading.Tasks; +using Apache.Arrow; +using Apache.Arrow.Ipc; + +namespace DeltaLake.Bridge +{ + internal sealed class RecordBatchReader : IArrowArrayStream + { + private readonly IEnumerator _enumerator; + + public RecordBatchReader(IEnumerable recordBatches, Schema schema) + { + _enumerator = recordBatches.GetEnumerator(); + Schema = schema; + } + + public Schema Schema { get; } + + public void Dispose() + { + _enumerator.Dispose(); + } + + public ValueTask ReadNextRecordBatchAsync(System.Threading.CancellationToken cancellationToken = default) + { + if (_enumerator.MoveNext()) + { + return ValueTask.FromResult(_enumerator.Current); + } + +#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. + return ValueTask.FromResult(default); +#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + } + } } \ No newline at end of file diff --git a/src/DeltaLake/Bridge/RentedByteArrayRef.cs b/src/DeltaLake/Bridge/RentedByteArrayRef.cs index 34dcc05..84e8847 100644 --- a/src/DeltaLake/Bridge/RentedByteArrayRef.cs +++ b/src/DeltaLake/Bridge/RentedByteArrayRef.cs @@ -1,36 +1,36 @@ -using System; -using System.Buffers; - -namespace DeltaLake.Bridge -{ - internal readonly struct RentedByteArrayRef : IDisposable - { - private readonly byte[] _array; - private readonly ArrayPool? _pool; - - public RentedByteArrayRef(ByteArrayRef byteArrayRef, byte[] array, ArrayPool? pool) - { - Ref = byteArrayRef; - _array = array; - _pool = pool; - } - - public static RentedByteArrayRef Empty { get; } = new(ByteArrayRef.Empty, Array.Empty(), null); - - public ByteArrayRef Ref { get; init; } - - - public readonly void Dispose() - { - Dispose(true); - } - - private readonly void Dispose(bool disposing) - { - if (disposing) - { - _pool?.Return(_array); - } - } - } +using System; +using System.Buffers; + +namespace DeltaLake.Bridge +{ + internal readonly struct RentedByteArrayRef : IDisposable + { + private readonly byte[] _array; + private readonly ArrayPool? _pool; + + public RentedByteArrayRef(ByteArrayRef byteArrayRef, byte[] array, ArrayPool? pool) + { + Ref = byteArrayRef; + _array = array; + _pool = pool; + } + + public static RentedByteArrayRef Empty { get; } = new(ByteArrayRef.Empty, Array.Empty(), null); + + public ByteArrayRef Ref { get; init; } + + + public readonly void Dispose() + { + Dispose(true); + } + + private readonly void Dispose(bool disposing) + { + if (disposing) + { + _pool?.Return(_array); + } + } + } } \ No newline at end of file diff --git a/src/DeltaLake/Bridge/Runtime.cs b/src/DeltaLake/Bridge/Runtime.cs index 03933b0..708f9f6 100644 --- a/src/DeltaLake/Bridge/Runtime.cs +++ b/src/DeltaLake/Bridge/Runtime.cs @@ -87,7 +87,7 @@ internal async Task
LoadTableAsync( { if (cancellationToken.IsCancellationRequested) { - Task.Run(() => tsc.TrySetCanceled(cancellationToken));; + Task.Run(() => tsc.TrySetCanceled(cancellationToken)); ; return; } @@ -139,7 +139,7 @@ internal async Task
CreateTableAsync(DeltaLake.Table.TableCreateOptions o { if (cancellationToken.IsCancellationRequested) { - Task.Run(() => tsc.TrySetCanceled(cancellationToken));; + Task.Run(() => tsc.TrySetCanceled(cancellationToken)); ; return; } diff --git a/src/DeltaLake/Bridge/Scope.cs b/src/DeltaLake/Bridge/Scope.cs index 982436a..286cf4f 100644 --- a/src/DeltaLake/Bridge/Scope.cs +++ b/src/DeltaLake/Bridge/Scope.cs @@ -1,171 +1,171 @@ -using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; - -namespace DeltaLake.Bridge -{ - /// - /// Disposable collection of items we need to keep alive while this object is in scope. - /// - internal sealed class Scope : IDisposable - { - private readonly List toKeepAlive = new(); - - /// - /// Create a byte array ref. - /// - /// Bytes to create from. - /// Created byte array ref. - public Interop.ByteArrayRef ByteArray(byte[]? bytes) - { - if (bytes == null || bytes.Length == 0) - { - return ByteArrayRef.Empty.Ref; - } - var val = new ByteArrayRef(bytes); - toKeepAlive.Add(val); - return val.Ref; - } - - /// - /// Create a byte array ref. - /// - /// Bytes to create from. - /// Created byte array ref. - public Interop.ByteArrayRef ByteArray(Memory bytes) - { - var val = new ByteArrayRef(bytes); - toKeepAlive.Add(val); - return val.Ref; - } - - /// - /// Create a UTF-8 byte array ref. - /// - /// String to create from. - /// Created byte array ref. - public Interop.ByteArrayRef ByteArray(string? str) - { - if (str == null || str.Length == 0) - { - return ByteArrayRef.Empty.Ref; - } - - var val = ByteArrayRef.RentUtf8(str); - toKeepAlive.Add(val); - toKeepAlive.Add(val.Ref); - return val.Ref.Ref; - } - - /// - /// Create a cancellation token. - /// - /// Cancellation token to create from. - /// Created cancellation token. - public unsafe Interop.CancellationToken* CancellationToken( - System.Threading.CancellationToken? token) - { - if (token == null) - { - return null; - } - - var val = DeltaLake.Bridge.CancellationToken.FromThreading(token.Value); - toKeepAlive.Add(val); - return val.Ptr; - } - - /// - /// Create a stable pointer to an object. - /// - /// Type of the object. - /// Object to get create pointer for. - /// Created pointer. - public unsafe T* Pointer(T value) - where T : unmanaged - { - var handle = GCHandle.Alloc(value, GCHandleType.Pinned); - toKeepAlive.Add(handle); - return (T*)handle.AddrOfPinnedObject(); - } - - /// - /// Creates a map with optional values - /// - /// Instance of runtime - /// Dictionary with nullable values - /// - public unsafe Interop.Map* OptionalDictionary(Runtime runtime, IReadOnlyCollection> value) - { - var map = Map.FromOptionalDictionary(runtime, value); - toKeepAlive.Add(map); - return map.Ref; - } - - /// - /// Creates a map - /// - /// Instance of runtime - /// Dictionary - /// - public unsafe Interop.Map* Dictionary(Runtime runtime, IReadOnlyCollection> value) - { - var map = Map.FromDictionary(runtime, value); - toKeepAlive.Add(map); - return map.Ref; - } - - - /// - /// Create a stable pointer to an object. - /// - /// Type of the object. - /// Object to get create pointer for. - /// Created pointer. - public unsafe T* ArrayPointer(T[] value) - where T : unmanaged - { - var handle = GCHandle.Alloc(value, GCHandleType.Pinned); - toKeepAlive.Add(handle); - return (T*)handle.AddrOfPinnedObject(); - } - - /// - /// Create function pointer for delegate. - /// - /// Delegate type. - /// Delegate to create pointer for. - /// Created pointer. - public IntPtr FunctionPointer(T func) - where T : Delegate - { - // The delegate seems to get collected before called sometimes even if we add "func" to - // the keep alive list. Delegates are supposed to be reference types, but their pointers - // seem unstable. So we're going to alloc a handle for it. We can't pin it though. - var handle = GCHandle.Alloc(func); - toKeepAlive.Add(handle); - return Marshal.GetFunctionPointerForDelegate(handle.Target!); - } - - /// - public void Dispose() - { - foreach (var v in toKeepAlive) - { - switch (v) - { - case GCHandle handle: - handle.Free(); - break; - case IDisposable disposable: - disposable.Dispose(); - break; - } - } - // This keep alive does nothing obviously, but it's good documentation to understand the - // purpose of this separate dispose call - GC.KeepAlive(toKeepAlive); - GC.SuppressFinalize(this); - } - } -} +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace DeltaLake.Bridge +{ + /// + /// Disposable collection of items we need to keep alive while this object is in scope. + /// + internal sealed class Scope : IDisposable + { + private readonly List toKeepAlive = new(); + + /// + /// Create a byte array ref. + /// + /// Bytes to create from. + /// Created byte array ref. + public Interop.ByteArrayRef ByteArray(byte[]? bytes) + { + if (bytes == null || bytes.Length == 0) + { + return ByteArrayRef.Empty.Ref; + } + var val = new ByteArrayRef(bytes); + toKeepAlive.Add(val); + return val.Ref; + } + + /// + /// Create a byte array ref. + /// + /// Bytes to create from. + /// Created byte array ref. + public Interop.ByteArrayRef ByteArray(Memory bytes) + { + var val = new ByteArrayRef(bytes); + toKeepAlive.Add(val); + return val.Ref; + } + + /// + /// Create a UTF-8 byte array ref. + /// + /// String to create from. + /// Created byte array ref. + public Interop.ByteArrayRef ByteArray(string? str) + { + if (str == null || str.Length == 0) + { + return ByteArrayRef.Empty.Ref; + } + + var val = ByteArrayRef.RentUtf8(str); + toKeepAlive.Add(val); + toKeepAlive.Add(val.Ref); + return val.Ref.Ref; + } + + /// + /// Create a cancellation token. + /// + /// Cancellation token to create from. + /// Created cancellation token. + public unsafe Interop.CancellationToken* CancellationToken( + System.Threading.CancellationToken? token) + { + if (token == null) + { + return null; + } + + var val = DeltaLake.Bridge.CancellationToken.FromThreading(token.Value); + toKeepAlive.Add(val); + return val.Ptr; + } + + /// + /// Create a stable pointer to an object. + /// + /// Type of the object. + /// Object to get create pointer for. + /// Created pointer. + public unsafe T* Pointer(T value) + where T : unmanaged + { + var handle = GCHandle.Alloc(value, GCHandleType.Pinned); + toKeepAlive.Add(handle); + return (T*)handle.AddrOfPinnedObject(); + } + + /// + /// Creates a map with optional values + /// + /// Instance of runtime + /// Dictionary with nullable values + /// + public unsafe Interop.Map* OptionalDictionary(Runtime runtime, IReadOnlyCollection> value) + { + var map = Map.FromOptionalDictionary(runtime, value); + toKeepAlive.Add(map); + return map.Ref; + } + + /// + /// Creates a map + /// + /// Instance of runtime + /// Dictionary + /// + public unsafe Interop.Map* Dictionary(Runtime runtime, IReadOnlyCollection> value) + { + var map = Map.FromDictionary(runtime, value); + toKeepAlive.Add(map); + return map.Ref; + } + + + /// + /// Create a stable pointer to an object. + /// + /// Type of the object. + /// Object to get create pointer for. + /// Created pointer. + public unsafe T* ArrayPointer(T[] value) + where T : unmanaged + { + var handle = GCHandle.Alloc(value, GCHandleType.Pinned); + toKeepAlive.Add(handle); + return (T*)handle.AddrOfPinnedObject(); + } + + /// + /// Create function pointer for delegate. + /// + /// Delegate type. + /// Delegate to create pointer for. + /// Created pointer. + public IntPtr FunctionPointer(T func) + where T : Delegate + { + // The delegate seems to get collected before called sometimes even if we add "func" to + // the keep alive list. Delegates are supposed to be reference types, but their pointers + // seem unstable. So we're going to alloc a handle for it. We can't pin it though. + var handle = GCHandle.Alloc(func); + toKeepAlive.Add(handle); + return Marshal.GetFunctionPointerForDelegate(handle.Target!); + } + + /// + public void Dispose() + { + foreach (var v in toKeepAlive) + { + switch (v) + { + case GCHandle handle: + handle.Free(); + break; + case IDisposable disposable: + disposable.Dispose(); + break; + } + } + // This keep alive does nothing obviously, but it's good documentation to understand the + // purpose of this separate dispose call + GC.KeepAlive(toKeepAlive); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/DeltaLake/Bridge/Table.cs b/src/DeltaLake/Bridge/Table.cs index f8bd841..34749c8 100644 --- a/src/DeltaLake/Bridge/Table.cs +++ b/src/DeltaLake/Bridge/Table.cs @@ -362,11 +362,12 @@ public async Task QueryAsync( else { var stream = CArrowArrayStreamImporter.ImportArrayStream((CArrowArrayStream*)success); - Task.Run(() => { - if (!tsc.TrySetResult(stream)) + Task.Run(() => { - stream.Dispose(); - } + if (!tsc.TrySetResult(stream)) + { + stream.Dispose(); + } }); } })); @@ -474,7 +475,7 @@ public async Task HistoryAsync(ulong limit, ICancellationToken cancellat { using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); var bytes = content.ToByteArray(); - Task.Run(() => tsc.TrySetResult(bytes));; + Task.Run(() => tsc.TrySetResult(bytes)); ; } })); diff --git a/src/DeltaLake/Errors/DeltaConfigurationException.cs b/src/DeltaLake/Errors/DeltaConfigurationException.cs index 7f7cead..fc7dd35 100644 --- a/src/DeltaLake/Errors/DeltaConfigurationException.cs +++ b/src/DeltaLake/Errors/DeltaConfigurationException.cs @@ -1,31 +1,31 @@ -using System; - -namespace DeltaLake.Errors -{ - /// - /// Represents a configuration error detected before interacting with the runtime - /// - public class DeltaConfigurationException : DeltaLakeException - { - - /// - /// Initializes a new instance of the DeltaConfigurationException class. - /// - /// The message that describes the error - public DeltaConfigurationException(string? message) - : base(message, 1000) - { - } - - /// - /// Initializes a new instance of the DeltaConfigurationException class - /// and a reference to the inner exception that is the cause of this exception - /// - /// The message that describes the error - /// The exception that is the cause of the current exception, or a null reference - public DeltaConfigurationException(string? message, Exception? innerException) - : base(message, 1000, innerException) - { - } - } +using System; + +namespace DeltaLake.Errors +{ + /// + /// Represents a configuration error detected before interacting with the runtime + /// + public class DeltaConfigurationException : DeltaLakeException + { + + /// + /// Initializes a new instance of the DeltaConfigurationException class. + /// + /// The message that describes the error + public DeltaConfigurationException(string? message) + : base(message, 1000) + { + } + + /// + /// Initializes a new instance of the DeltaConfigurationException class + /// and a reference to the inner exception that is the cause of this exception + /// + /// The message that describes the error + /// The exception that is the cause of the current exception, or a null reference + public DeltaConfigurationException(string? message, Exception? innerException) + : base(message, 1000, innerException) + { + } + } } \ No newline at end of file diff --git a/src/DeltaLake/Errors/DeltaLakeException.cs b/src/DeltaLake/Errors/DeltaLakeException.cs index 2d8705e..1ce85a0 100644 --- a/src/DeltaLake/Errors/DeltaLakeException.cs +++ b/src/DeltaLake/Errors/DeltaLakeException.cs @@ -1,39 +1,39 @@ -using System; - -namespace DeltaLake.Errors -{ - /// - /// Represents an error that occurs during interaction with the underlying runtime - /// - public class DeltaLakeException : Exception - { - /// - /// Initializes a new instance of the DeltaLakeException class. - /// - /// The message that describes the error - /// The code that describes the error - public DeltaLakeException(string? message, int code) : base(message) - { - ErrorCode = code; - } - - - /// - /// Initializes a new instance of the DeltaLakeException class - /// and a reference to the inner exception that is the cause of this exception - /// - /// The message that describes the error - /// The code that describes the error - /// The exception that is the cause of the current exception, or a null reference - public DeltaLakeException(string? message, int code, Exception? innerException) - : base(message, innerException) - { - ErrorCode = code; - } - - /// - /// A unique error code coming from the delta lake runtime - /// - public int ErrorCode { get; } - } +using System; + +namespace DeltaLake.Errors +{ + /// + /// Represents an error that occurs during interaction with the underlying runtime + /// + public class DeltaLakeException : Exception + { + /// + /// Initializes a new instance of the DeltaLakeException class. + /// + /// The message that describes the error + /// The code that describes the error + public DeltaLakeException(string? message, int code) : base(message) + { + ErrorCode = code; + } + + + /// + /// Initializes a new instance of the DeltaLakeException class + /// and a reference to the inner exception that is the cause of this exception + /// + /// The message that describes the error + /// The code that describes the error + /// The exception that is the cause of the current exception, or a null reference + public DeltaLakeException(string? message, int code, Exception? innerException) + : base(message, innerException) + { + ErrorCode = code; + } + + /// + /// A unique error code coming from the delta lake runtime + /// + public int ErrorCode { get; } + } } \ No newline at end of file diff --git a/src/DeltaLake/Errors/DeltaRuntimeException.cs b/src/DeltaLake/Errors/DeltaRuntimeException.cs index 51efb5a..fa39625 100644 --- a/src/DeltaLake/Errors/DeltaRuntimeException.cs +++ b/src/DeltaLake/Errors/DeltaRuntimeException.cs @@ -1,59 +1,59 @@ -using System; -using DeltaLake.Bridge; - -namespace DeltaLake.Errors -{ - /// - /// Represents an error that occurs during interaction with the underlying runtime - /// - public class DeltaRuntimeException : DeltaLakeException - { - /// - /// Initializes a new instance of the DeltaRuntimeException class. - /// - /// The message that describes the error - /// The code that describes the error - public DeltaRuntimeException(string? message, int code) - : base(message, code) - { - } - - - /// - /// Initializes a new instance of the DeltaRuntimeException class - /// and a reference to the inner exception that is the cause of this exception - /// - /// The message that describes the error - /// The code that describes the error - /// The exception that is the cause of the current exception, or a null reference - public DeltaRuntimeException(string? message, int code, Exception? innerException) - : base(message, code, innerException) - { - } - - /// - /// Consumes and frees the - /// - /// Pointer to the unmanaged runtime - /// Pointer to the unmanaged error - /// A new DeltaRuntimeException instance - /// Throw when error is null - internal unsafe static DeltaRuntimeException FromDeltaTableError(Bridge.Interop.Runtime* runtime, Bridge.Interop.DeltaTableError* error) - { - if (error == null) - { - throw new ArgumentNullException(nameof(error)); - } - - try - { - var message = ByteArrayRef.StrictUTF8.GetString(error->error.data, (int)error->error.size); - return new DeltaRuntimeException(message, (int)error->code); - } - finally - { - Bridge.Interop.Methods.error_free(runtime, error); - } - } - } +using System; +using DeltaLake.Bridge; + +namespace DeltaLake.Errors +{ + /// + /// Represents an error that occurs during interaction with the underlying runtime + /// + public class DeltaRuntimeException : DeltaLakeException + { + /// + /// Initializes a new instance of the DeltaRuntimeException class. + /// + /// The message that describes the error + /// The code that describes the error + public DeltaRuntimeException(string? message, int code) + : base(message, code) + { + } + + + /// + /// Initializes a new instance of the DeltaRuntimeException class + /// and a reference to the inner exception that is the cause of this exception + /// + /// The message that describes the error + /// The code that describes the error + /// The exception that is the cause of the current exception, or a null reference + public DeltaRuntimeException(string? message, int code, Exception? innerException) + : base(message, code, innerException) + { + } + + /// + /// Consumes and frees the + /// + /// Pointer to the unmanaged runtime + /// Pointer to the unmanaged error + /// A new DeltaRuntimeException instance + /// Throw when error is null + internal unsafe static DeltaRuntimeException FromDeltaTableError(Bridge.Interop.Runtime* runtime, Bridge.Interop.DeltaTableError* error) + { + if (error == null) + { + throw new ArgumentNullException(nameof(error)); + } + + try + { + var message = ByteArrayRef.StrictUTF8.GetString(error->error.data, (int)error->error.size); + return new DeltaRuntimeException(message, (int)error->code); + } + finally + { + Bridge.Interop.Methods.error_free(runtime, error); + } + } + } } \ No newline at end of file diff --git a/src/DeltaLake/Runtime/DeltaRuntime.cs b/src/DeltaLake/Runtime/DeltaRuntime.cs index 54faec9..3246e57 100644 --- a/src/DeltaLake/Runtime/DeltaRuntime.cs +++ b/src/DeltaLake/Runtime/DeltaRuntime.cs @@ -1,40 +1,40 @@ -using System; - -namespace DeltaLake.Runtime -{ - - /// - /// Represents the runtime - /// - public sealed class DeltaRuntime : IDisposable - { - /// - /// - /// - /// - public DeltaRuntime(RuntimeOptions options) - { - Runtime = new Bridge.Runtime(options); - } - - /// - /// Gets the runtime. - /// - internal Bridge.Runtime Runtime { get; private init; } - - /// - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - private void Dispose(bool _disposing) - { - if (_disposing) - { - Runtime.Dispose(); - } - } - } +using System; + +namespace DeltaLake.Runtime +{ + + /// + /// Represents the runtime + /// + public sealed class DeltaRuntime : IDisposable + { + /// + /// + /// + /// + public DeltaRuntime(RuntimeOptions options) + { + Runtime = new Bridge.Runtime(options); + } + + /// + /// Gets the runtime. + /// + internal Bridge.Runtime Runtime { get; private init; } + + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + private void Dispose(bool _disposing) + { + if (_disposing) + { + Runtime.Dispose(); + } + } + } } \ No newline at end of file diff --git a/src/DeltaLake/Runtime/RuntimeOptions.cs b/src/DeltaLake/Runtime/RuntimeOptions.cs index 2516628..d50f53a 100644 --- a/src/DeltaLake/Runtime/RuntimeOptions.cs +++ b/src/DeltaLake/Runtime/RuntimeOptions.cs @@ -1,14 +1,14 @@ -namespace DeltaLake.Runtime -{ - - /// - /// - /// - public sealed class RuntimeOptions - { - /// - /// Default instance of runtime options - /// - public static RuntimeOptions Default { get; } = new RuntimeOptions(); - } +namespace DeltaLake.Runtime +{ + + /// + /// + /// + public sealed class RuntimeOptions + { + /// + /// Default instance of runtime options + /// + public static RuntimeOptions Default { get; } = new RuntimeOptions(); + } } \ No newline at end of file diff --git a/src/DeltaLake/Table/CommitInfo.cs b/src/DeltaLake/Table/CommitInfo.cs index 6baf1c2..4235c72 100644 --- a/src/DeltaLake/Table/CommitInfo.cs +++ b/src/DeltaLake/Table/CommitInfo.cs @@ -1,70 +1,70 @@ -using System.Collections.Generic; -using System.Text.Json.Nodes; -using System.Text.Json.Serialization; - -namespace DeltaLake.Table -{ - /// - /// CommitInfo for history command - /// -#pragma warning disable CA2227 // Collection properties should be read only - public class CommitInfo - { - /// - /// Timestamp in millis when the commit was created - /// - [JsonPropertyName("timestamp")] - public long? Timestamp { get; set; } - - /// - /// Id of the user invoking the commit[] - /// - [JsonPropertyName("user_id")] - - public string? UserId { get; set; } - /// - /// Name of the user invoking the commit - /// - [JsonPropertyName("user_name")] - public string? UserName { get; set; } - /// - /// The operation performed during the - /// - [JsonPropertyName("operation")] - public string? Operation { get; set; } - - /// - /// Parameters used for table operation - /// - [JsonPropertyName("operation_parameters")] - - public Dictionary? OperationParameters { get; set; } - - /// - /// Version of the table when the operation was started - /// - [JsonPropertyName("read_version")] - public long? ReadVersion { get; set; } - /// - /// The isolation level of the commit - /// - [JsonPropertyName("isolation_level")] - public string? IsolationLevel { get; set; } - /// - /// Undocumented - /// - [JsonPropertyName("is_blind_append")] - public bool? IsBlindAppend { get; set; } - /// - /// Delta engine which created the commit. - /// - [JsonPropertyName("engine_info")] - public string? EngineInfo { get; set; } - /// - /// Additional provenance information for the commit - /// - [JsonPropertyName("info")] - public Dictionary? Info { get; set; } - } -#pragma warning restore CA2227 // Collection properties should be read only +using System.Collections.Generic; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace DeltaLake.Table +{ + /// + /// CommitInfo for history command + /// +#pragma warning disable CA2227 // Collection properties should be read only + public class CommitInfo + { + /// + /// Timestamp in millis when the commit was created + /// + [JsonPropertyName("timestamp")] + public long? Timestamp { get; set; } + + /// + /// Id of the user invoking the commit[] + /// + [JsonPropertyName("user_id")] + + public string? UserId { get; set; } + /// + /// Name of the user invoking the commit + /// + [JsonPropertyName("user_name")] + public string? UserName { get; set; } + /// + /// The operation performed during the + /// + [JsonPropertyName("operation")] + public string? Operation { get; set; } + + /// + /// Parameters used for table operation + /// + [JsonPropertyName("operation_parameters")] + + public Dictionary? OperationParameters { get; set; } + + /// + /// Version of the table when the operation was started + /// + [JsonPropertyName("read_version")] + public long? ReadVersion { get; set; } + /// + /// The isolation level of the commit + /// + [JsonPropertyName("isolation_level")] + public string? IsolationLevel { get; set; } + /// + /// Undocumented + /// + [JsonPropertyName("is_blind_append")] + public bool? IsBlindAppend { get; set; } + /// + /// Delta engine which created the commit. + /// + [JsonPropertyName("engine_info")] + public string? EngineInfo { get; set; } + /// + /// Additional provenance information for the commit + /// + [JsonPropertyName("info")] + public Dictionary? Info { get; set; } + } +#pragma warning restore CA2227 // Collection properties should be read only } \ No newline at end of file diff --git a/src/DeltaLake/Table/DeltaTable.cs b/src/DeltaLake/Table/DeltaTable.cs index f8d32d0..d19dca4 100644 --- a/src/DeltaLake/Table/DeltaTable.cs +++ b/src/DeltaLake/Table/DeltaTable.cs @@ -1,357 +1,357 @@ -using System; -using System.Collections.Generic; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using Apache.Arrow; -using Apache.Arrow.Ipc; -using DeltaLake.Errors; -using DeltaLake.Runtime; - -namespace DeltaLake.Table -{ - /// - /// Represents a deltalake table - /// - public sealed class DeltaTable : IDisposable - { - private readonly Bridge.Table _table; - - private DeltaTable(Bridge.Table table) - { - _table = table; - } - - /// - /// - /// - /// - /// - /// - /// A cancellation token - /// A new delta table loaded at the latest version - public static async Task LoadAsync( - DeltaRuntime runtime, - string location, - TableOptions options, - CancellationToken cancellationToken) - { - var table = await runtime.Runtime.LoadTableAsync(location, options, cancellationToken).ConfigureAwait(false); - return new DeltaTable(table); - } - - /// - /// - /// - /// - /// - /// - /// A cancellation token - /// A new delta table loaded at the latest version - public static async Task LoadAsync( - DeltaRuntime runtime, - Memory uri, - TableOptions options, - CancellationToken cancellationToken) - { - var table = await runtime.Runtime.LoadTableAsync(uri, options, cancellationToken).ConfigureAwait(false); - return new DeltaTable(table); - } - - /// - /// Creates a new delta table from the specified options - /// - /// - /// - /// A cancellation token - /// New DeltaTable - public static async Task CreateAsync( - DeltaRuntime runtime, - TableCreateOptions options, - CancellationToken cancellationToken) - { - var table = await runtime.Runtime.CreateTableAsync(options, cancellationToken).ConfigureAwait(false); - return new DeltaTable(table); - } - - /// - /// Retrieves the current table files - /// - /// list of files - public string[] Files() - { - return _table.Files(); - } - - /// - /// Retrieves the current table file uris - /// - /// list of paths - public string[] FileUris() - { - return _table.FileUris(); - } - - /// - /// Retrieves the current table location - /// - /// table location - public string Location() - { - return _table.Uri(); - } - - /// - /// Retrieves the current table version - /// - /// - public ulong Version() - { - return (ulong)_table.Version(); - } - - /// - /// Loads table at specific version - /// - /// desired version - /// A cancellation token - /// - public Task LoadVersionAsync(ulong version, CancellationToken cancellationToken) - { - return _table.LoadVersionAsync(version, cancellationToken); - } - - /// - /// Returns the table schema - /// - /// - public Apache.Arrow.Schema Schema() - { - return _table.Schema(); - } - - /// - /// Inserts a record batch into the table based upon the provided options - /// - /// A collection of records to insert - /// The associated for the - /// - /// - /// - public async Task InsertAsync( - IReadOnlyCollection records, - Schema schema, - InsertOptions options, - CancellationToken cancellationToken) - { - if (!options.IsValid) - { - throw new DeltaConfigurationException( - "Invalid InsertOptions", - new ArgumentException("configuration is invalid", nameof(options))); - } - - await _table.InsertAsync(records, schema, options, cancellationToken).ConfigureAwait(false); - } - - /// - /// Inserts a record batch into the table based upon the provided options - /// - /// A collection of records to insert - /// - /// - /// - public async Task InsertAsync( - IArrowArrayStream records, - InsertOptions options, - CancellationToken cancellationToken) - { - - if (!options.IsValid) - { - throw new DeltaConfigurationException( - "Invalid InsertOptions", - new ArgumentException("configuration is invalid", nameof(options))); - } - - await _table.InsertAsync(records, options, cancellationToken).ConfigureAwait(false); - } - - /// - /// Returns the table history - /// - /// Optional maximum amount of history to return - /// - /// An array of - public async Task HistoryAsync(ulong? limit, CancellationToken cancellationToken) - { - var content = await _table.HistoryAsync(limit ?? 0, cancellationToken).ConfigureAwait(false); - return System.Text.Json.JsonSerializer.Deserialize(content) ?? System.Array.Empty(); - } - - /// - /// Adds constraints and custom metadata to a table - /// - /// A collection of key and values representing columns and constraints - /// A collection of key and values representing columns and metadata - /// - /// - public async Task AddConstraintsAsync( - IReadOnlyDictionary constraints, - IReadOnlyDictionary? customMetadata, - CancellationToken cancellationToken) - { - await _table.AddConstraintAsync(constraints, customMetadata, cancellationToken).ConfigureAwait(false); - } - - /// - /// Adds constraints and custom metadata to a table - /// - /// A collection of key and values representing columns and constraints - /// - /// - public Task AddConstraintsAsync( - IReadOnlyDictionary constraints, - CancellationToken cancellationToken) - { - return AddConstraintsAsync(constraints, null, cancellationToken); - } - - /// - /// Updates table to specific or latest version - /// - /// Optional maximum version - /// - /// - public Task UpdateIncrementalAsync(long? maxVersion, CancellationToken cancellationToken) - { - return _table.UpdateIncrementalAsync(maxVersion, cancellationToken); - } - - /// - /// Loads table at a specific point in time - /// - /// desired point in time - /// A cancellation token - /// - public Task LoadDateTimeAsync(DateTimeOffset timestamp, CancellationToken cancellationToken) - { - return LoadDateTimeAsync(timestamp.ToUnixTimeMilliseconds(), cancellationToken); - } - - /// - /// Loads table at a specific point in time - /// - /// desired point in time in unix milliseconds - /// A cancellation token - /// - public Task LoadDateTimeAsync(long timestampMilliseconds, CancellationToken cancellationToken) - { - return _table.LoadTimestampAsync(timestampMilliseconds, cancellationToken); - } - - /// - /// Merges a collection of into the delta table - /// - /// A SQL MERGE statement - /// A collection of record batches - /// The schema of the collection - /// - /// - /// - public async Task MergeAsync(string query, IReadOnlyCollection records, Schema schema, CancellationToken cancellationToken) - { - await _table.MergeAsync(query, records, schema, cancellationToken).ConfigureAwait(false); - } - - /// - /// Returns table metadata - /// - /// - public TableMetadata Metadata() - { - return _table.Metadata(); - } - - /// - /// Returns minimum reader and writer versions - /// - /// - public ProtocolInfo ProtocolVersions() - { - return _table.ProtocolVersions(); - } - - /// - /// Restores a table using the options provided - /// - /// Restore options - /// - /// - public Task RestoreAsync(RestoreOptions options, CancellationToken cancellationToken) - { - return _table.RestoreAsync(options, cancellationToken); - } - - /// - /// Returns json serialized statistics about the update issued against the table - /// - /// Sql UPDATE statement against the table - /// - /// json data - public Task UpdateAsync(string query, CancellationToken cancellationToken) - { - return _table.UpdateAsync(query, cancellationToken); - } - - /// - /// Accepts a predicate and deletes matching items from the table - /// - /// The criteria for deletion... ex color = 'Blue' AND ts > 10000 - /// - /// - public Task DeleteAsync(string predicate, CancellationToken cancellationToken) - { - return _table.DeleteAsync(predicate, cancellationToken); - } - - /// - /// Accepts a predicate and deletes matching items from the table - /// - /// - /// - public Task DeleteAsync(CancellationToken cancellationToken) - { - return _table.DeleteAsync(string.Empty, cancellationToken); - } - - /// - /// Issue a select query against a delta table - /// - /// A select query - /// - /// A collection of record batches representing the query results - public async IAsyncEnumerable QueryAsync( - SelectQuery query, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - var result = await _table.QueryAsync(query.Query, query.TableAlias, cancellationToken).ConfigureAwait(false); - while (true) - { - var rb = await result.ReadNextRecordBatchAsync(cancellationToken).ConfigureAwait(false); - if (rb == null) - { - yield break; - } - - yield return rb; - } - } - - /// - public void Dispose() - { - _table.Dispose(); - } - } +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using DeltaLake.Errors; +using DeltaLake.Runtime; + +namespace DeltaLake.Table +{ + /// + /// Represents a deltalake table + /// + public sealed class DeltaTable : IDisposable + { + private readonly Bridge.Table _table; + + private DeltaTable(Bridge.Table table) + { + _table = table; + } + + /// + /// + /// + /// + /// + /// + /// A cancellation token + /// A new delta table loaded at the latest version + public static async Task LoadAsync( + DeltaRuntime runtime, + string location, + TableOptions options, + CancellationToken cancellationToken) + { + var table = await runtime.Runtime.LoadTableAsync(location, options, cancellationToken).ConfigureAwait(false); + return new DeltaTable(table); + } + + /// + /// + /// + /// + /// + /// + /// A cancellation token + /// A new delta table loaded at the latest version + public static async Task LoadAsync( + DeltaRuntime runtime, + Memory uri, + TableOptions options, + CancellationToken cancellationToken) + { + var table = await runtime.Runtime.LoadTableAsync(uri, options, cancellationToken).ConfigureAwait(false); + return new DeltaTable(table); + } + + /// + /// Creates a new delta table from the specified options + /// + /// + /// + /// A cancellation token + /// New DeltaTable + public static async Task CreateAsync( + DeltaRuntime runtime, + TableCreateOptions options, + CancellationToken cancellationToken) + { + var table = await runtime.Runtime.CreateTableAsync(options, cancellationToken).ConfigureAwait(false); + return new DeltaTable(table); + } + + /// + /// Retrieves the current table files + /// + /// list of files + public string[] Files() + { + return _table.Files(); + } + + /// + /// Retrieves the current table file uris + /// + /// list of paths + public string[] FileUris() + { + return _table.FileUris(); + } + + /// + /// Retrieves the current table location + /// + /// table location + public string Location() + { + return _table.Uri(); + } + + /// + /// Retrieves the current table version + /// + /// + public ulong Version() + { + return (ulong)_table.Version(); + } + + /// + /// Loads table at specific version + /// + /// desired version + /// A cancellation token + /// + public Task LoadVersionAsync(ulong version, CancellationToken cancellationToken) + { + return _table.LoadVersionAsync(version, cancellationToken); + } + + /// + /// Returns the table schema + /// + /// + public Apache.Arrow.Schema Schema() + { + return _table.Schema(); + } + + /// + /// Inserts a record batch into the table based upon the provided options + /// + /// A collection of records to insert + /// The associated for the + /// + /// + /// + public async Task InsertAsync( + IReadOnlyCollection records, + Schema schema, + InsertOptions options, + CancellationToken cancellationToken) + { + if (!options.IsValid) + { + throw new DeltaConfigurationException( + "Invalid InsertOptions", + new ArgumentException("configuration is invalid", nameof(options))); + } + + await _table.InsertAsync(records, schema, options, cancellationToken).ConfigureAwait(false); + } + + /// + /// Inserts a record batch into the table based upon the provided options + /// + /// A collection of records to insert + /// + /// + /// + public async Task InsertAsync( + IArrowArrayStream records, + InsertOptions options, + CancellationToken cancellationToken) + { + + if (!options.IsValid) + { + throw new DeltaConfigurationException( + "Invalid InsertOptions", + new ArgumentException("configuration is invalid", nameof(options))); + } + + await _table.InsertAsync(records, options, cancellationToken).ConfigureAwait(false); + } + + /// + /// Returns the table history + /// + /// Optional maximum amount of history to return + /// + /// An array of + public async Task HistoryAsync(ulong? limit, CancellationToken cancellationToken) + { + var content = await _table.HistoryAsync(limit ?? 0, cancellationToken).ConfigureAwait(false); + return System.Text.Json.JsonSerializer.Deserialize(content) ?? System.Array.Empty(); + } + + /// + /// Adds constraints and custom metadata to a table + /// + /// A collection of key and values representing columns and constraints + /// A collection of key and values representing columns and metadata + /// + /// + public async Task AddConstraintsAsync( + IReadOnlyDictionary constraints, + IReadOnlyDictionary? customMetadata, + CancellationToken cancellationToken) + { + await _table.AddConstraintAsync(constraints, customMetadata, cancellationToken).ConfigureAwait(false); + } + + /// + /// Adds constraints and custom metadata to a table + /// + /// A collection of key and values representing columns and constraints + /// + /// + public Task AddConstraintsAsync( + IReadOnlyDictionary constraints, + CancellationToken cancellationToken) + { + return AddConstraintsAsync(constraints, null, cancellationToken); + } + + /// + /// Updates table to specific or latest version + /// + /// Optional maximum version + /// + /// + public Task UpdateIncrementalAsync(long? maxVersion, CancellationToken cancellationToken) + { + return _table.UpdateIncrementalAsync(maxVersion, cancellationToken); + } + + /// + /// Loads table at a specific point in time + /// + /// desired point in time + /// A cancellation token + /// + public Task LoadDateTimeAsync(DateTimeOffset timestamp, CancellationToken cancellationToken) + { + return LoadDateTimeAsync(timestamp.ToUnixTimeMilliseconds(), cancellationToken); + } + + /// + /// Loads table at a specific point in time + /// + /// desired point in time in unix milliseconds + /// A cancellation token + /// + public Task LoadDateTimeAsync(long timestampMilliseconds, CancellationToken cancellationToken) + { + return _table.LoadTimestampAsync(timestampMilliseconds, cancellationToken); + } + + /// + /// Merges a collection of into the delta table + /// + /// A SQL MERGE statement + /// A collection of record batches + /// The schema of the collection + /// + /// + /// + public async Task MergeAsync(string query, IReadOnlyCollection records, Schema schema, CancellationToken cancellationToken) + { + await _table.MergeAsync(query, records, schema, cancellationToken).ConfigureAwait(false); + } + + /// + /// Returns table metadata + /// + /// + public TableMetadata Metadata() + { + return _table.Metadata(); + } + + /// + /// Returns minimum reader and writer versions + /// + /// + public ProtocolInfo ProtocolVersions() + { + return _table.ProtocolVersions(); + } + + /// + /// Restores a table using the options provided + /// + /// Restore options + /// + /// + public Task RestoreAsync(RestoreOptions options, CancellationToken cancellationToken) + { + return _table.RestoreAsync(options, cancellationToken); + } + + /// + /// Returns json serialized statistics about the update issued against the table + /// + /// Sql UPDATE statement against the table + /// + /// json data + public Task UpdateAsync(string query, CancellationToken cancellationToken) + { + return _table.UpdateAsync(query, cancellationToken); + } + + /// + /// Accepts a predicate and deletes matching items from the table + /// + /// The criteria for deletion... ex color = 'Blue' AND ts > 10000 + /// + /// + public Task DeleteAsync(string predicate, CancellationToken cancellationToken) + { + return _table.DeleteAsync(predicate, cancellationToken); + } + + /// + /// Accepts a predicate and deletes matching items from the table + /// + /// + /// + public Task DeleteAsync(CancellationToken cancellationToken) + { + return _table.DeleteAsync(string.Empty, cancellationToken); + } + + /// + /// Issue a select query against a delta table + /// + /// A select query + /// + /// A collection of record batches representing the query results + public async IAsyncEnumerable QueryAsync( + SelectQuery query, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var result = await _table.QueryAsync(query.Query, query.TableAlias, cancellationToken).ConfigureAwait(false); + while (true) + { + var rb = await result.ReadNextRecordBatchAsync(cancellationToken).ConfigureAwait(false); + if (rb == null) + { + yield break; + } + + yield return rb; + } + } + + /// + public void Dispose() + { + _table.Dispose(); + } + } } \ No newline at end of file diff --git a/src/DeltaLake/Table/InsertOptions.cs b/src/DeltaLake/Table/InsertOptions.cs index b637d75..4db00bd 100644 --- a/src/DeltaLake/Table/InsertOptions.cs +++ b/src/DeltaLake/Table/InsertOptions.cs @@ -1,34 +1,34 @@ -namespace DeltaLake.Table -{ - /// - /// Options for inserting data into a table - /// - public class InsertOptions - { - /// - /// Predicate for insertion. - /// Represents the WHERE clause, not including WHERE - /// - public string? Predicate { get; init; } - - /// - /// - /// - public SaveMode SaveMode { get; init; } - - /// - /// Maximum number of rows to write per row group - /// - public ulong MaxRowsPerGroup { get; init; } = 100; - - /// - /// Overwrite schema with schema from record batch - /// - public bool OverwriteSchema { get; init; } - - /// - /// Indicates a valid configuration - /// - public bool IsValid => !(this.SaveMode == SaveMode.Append && OverwriteSchema); - } +namespace DeltaLake.Table +{ + /// + /// Options for inserting data into a table + /// + public class InsertOptions + { + /// + /// Predicate for insertion. + /// Represents the WHERE clause, not including WHERE + /// + public string? Predicate { get; init; } + + /// + /// + /// + public SaveMode SaveMode { get; init; } + + /// + /// Maximum number of rows to write per row group + /// + public ulong MaxRowsPerGroup { get; init; } = 100; + + /// + /// Overwrite schema with schema from record batch + /// + public bool OverwriteSchema { get; init; } + + /// + /// Indicates a valid configuration + /// + public bool IsValid => !(this.SaveMode == SaveMode.Append && OverwriteSchema); + } } \ No newline at end of file diff --git a/src/DeltaLake/Table/ProtocolInfo.cs b/src/DeltaLake/Table/ProtocolInfo.cs index 9f02d76..c1d7ddd 100644 --- a/src/DeltaLake/Table/ProtocolInfo.cs +++ b/src/DeltaLake/Table/ProtocolInfo.cs @@ -1,20 +1,20 @@ -namespace DeltaLake.Table -{ - /// - /// Protocol information for a delta table - /// -#pragma warning disable CA1815 // Override equals and operator equals on value types - public readonly struct ProtocolInfo -#pragma warning restore CA1815 // Override equals and operator equals on value types - { - /// - /// Table minimum reader version - /// - public int MinimumReaderVersion { get; init; } - - /// - /// Table minimum writer version - /// - public int MinimumWriterVersion { get; init; } - } +namespace DeltaLake.Table +{ + /// + /// Protocol information for a delta table + /// +#pragma warning disable CA1815 // Override equals and operator equals on value types + public readonly struct ProtocolInfo +#pragma warning restore CA1815 // Override equals and operator equals on value types + { + /// + /// Table minimum reader version + /// + public int MinimumReaderVersion { get; init; } + + /// + /// Table minimum writer version + /// + public int MinimumWriterVersion { get; init; } + } } \ No newline at end of file diff --git a/src/DeltaLake/Table/RestoreOptions.cs b/src/DeltaLake/Table/RestoreOptions.cs index 0f060d2..4d90957 100644 --- a/src/DeltaLake/Table/RestoreOptions.cs +++ b/src/DeltaLake/Table/RestoreOptions.cs @@ -1,36 +1,36 @@ -using System; -using System.Collections.Generic; - -namespace DeltaLake.Table -{ - /// - /// Options for restoring a delta table - /// - public class RestoreOptions - { - /// - /// Optional version. Mutually excluse with - /// - public ulong? Version { get; init; } - - /// - /// Timestamp restore point. Mutually exclusive with - /// - public DateTimeOffset? Timestamp { get; init; } - - /// - /// Ignore missing files during restoration - /// - public bool IgnoreMissingFiles { get; init; } - - /// - /// Allow protocol downgrades - /// - public bool ProtocolDowngradeAllowed { get; init; } - - /// - /// Custom metadata - /// - public Dictionary CustomMetadata { get; init; } = new Dictionary(); - } +using System; +using System.Collections.Generic; + +namespace DeltaLake.Table +{ + /// + /// Options for restoring a delta table + /// + public class RestoreOptions + { + /// + /// Optional version. Mutually excluse with + /// + public ulong? Version { get; init; } + + /// + /// Timestamp restore point. Mutually exclusive with + /// + public DateTimeOffset? Timestamp { get; init; } + + /// + /// Ignore missing files during restoration + /// + public bool IgnoreMissingFiles { get; init; } + + /// + /// Allow protocol downgrades + /// + public bool ProtocolDowngradeAllowed { get; init; } + + /// + /// Custom metadata + /// + public Dictionary CustomMetadata { get; init; } = new Dictionary(); + } } \ No newline at end of file diff --git a/src/DeltaLake/Table/SaveMode.cs b/src/DeltaLake/Table/SaveMode.cs index eb317b7..139c0d1 100644 --- a/src/DeltaLake/Table/SaveMode.cs +++ b/src/DeltaLake/Table/SaveMode.cs @@ -1,25 +1,25 @@ -namespace DeltaLake.Table -{ - /// - /// Save mode for the created delta table - /// - public enum SaveMode - { - /// - /// Append to the table - /// - Append, - /// - /// The target location will be overwritten. - /// - Overwrite, - /// - /// If files exist for the target, the operation must fail. - /// - ErrorIfExists, - /// - /// /// If files exist for the target, the operation must not proceed or change any data. - /// - Ignore, - } +namespace DeltaLake.Table +{ + /// + /// Save mode for the created delta table + /// + public enum SaveMode + { + /// + /// Append to the table + /// + Append, + /// + /// The target location will be overwritten. + /// + Overwrite, + /// + /// If files exist for the target, the operation must fail. + /// + ErrorIfExists, + /// + /// /// If files exist for the target, the operation must not proceed or change any data. + /// + Ignore, + } } \ No newline at end of file diff --git a/src/DeltaLake/Table/SelectQuery.cs b/src/DeltaLake/Table/SelectQuery.cs index 20064bf..abae8aa 100644 --- a/src/DeltaLake/Table/SelectQuery.cs +++ b/src/DeltaLake/Table/SelectQuery.cs @@ -1,30 +1,30 @@ -namespace DeltaLake.Table -{ - /// - /// Strongly types query wrapper for select queries - /// - public class SelectQuery - { - /// - /// Create a query with the provided query and default table alias 'deltatable' - /// - /// A SQL SELECT query - /// Alias for the table used in the select query - - public SelectQuery(string query, string tableAlias = "deltatable") - { - Query = query; - TableAlias = tableAlias; - } - - /// - /// Select query in the form of a sql select statement - /// - public string Query { get; } - - /// - /// The name for the table used in the select query - /// - public string TableAlias { get; init; } - } +namespace DeltaLake.Table +{ + /// + /// Strongly types query wrapper for select queries + /// + public class SelectQuery + { + /// + /// Create a query with the provided query and default table alias 'deltatable' + /// + /// A SQL SELECT query + /// Alias for the table used in the select query + + public SelectQuery(string query, string tableAlias = "deltatable") + { + Query = query; + TableAlias = tableAlias; + } + + /// + /// Select query in the form of a sql select statement + /// + public string Query { get; } + + /// + /// The name for the table used in the select query + /// + public string TableAlias { get; init; } + } } \ No newline at end of file diff --git a/src/DeltaLake/Table/TableMetadata.cs b/src/DeltaLake/Table/TableMetadata.cs index a4a913b..9dd20ba 100644 --- a/src/DeltaLake/Table/TableMetadata.cs +++ b/src/DeltaLake/Table/TableMetadata.cs @@ -116,7 +116,8 @@ private unsafe static Dictionary KeyValueToDictionary(Bridge.Int var toReturn = new Dictionary(); foreach (var (key, value) in KeyValueToDictionaryNullable(kvs).Where(x => x.Value != null)) { - if (value != null) { + if (value != null) + { toReturn[key] = value; } } diff --git a/src/DeltaLake/Table/TableOptions.cs b/src/DeltaLake/Table/TableOptions.cs index 49749b5..bc32552 100644 --- a/src/DeltaLake/Table/TableOptions.cs +++ b/src/DeltaLake/Table/TableOptions.cs @@ -1,30 +1,30 @@ -using System.Collections.Generic; - -namespace DeltaLake.Table -{ - /// - /// Options for creating a new delta table - /// - public class TableOptions - { - /// - /// Optional version of the table to load - /// - public ulong? Version { get; set; } - - /// - /// A map of string options - /// - public Dictionary StorageOptions { get; } = new Dictionary(); - - /// - /// Whether or not to load files when building the table - /// - public bool WithoutFiles { get; set; } - - /// - /// Buffer size for log files - /// - public uint? LogBufferSize { get; set; } - } +using System.Collections.Generic; + +namespace DeltaLake.Table +{ + /// + /// Options for creating a new delta table + /// + public class TableOptions + { + /// + /// Optional version of the table to load + /// + public ulong? Version { get; set; } + + /// + /// A map of string options + /// + public Dictionary StorageOptions { get; } = new Dictionary(); + + /// + /// Whether or not to load files when building the table + /// + public bool WithoutFiles { get; set; } + + /// + /// Buffer size for log files + /// + public uint? LogBufferSize { get; set; } + } } \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Runtime/RuntimeTests.cs b/tests/DeltaLake.Tests/Runtime/RuntimeTests.cs index 6658fed..9a2a51e 100644 --- a/tests/DeltaLake.Tests/Runtime/RuntimeTests.cs +++ b/tests/DeltaLake.Tests/Runtime/RuntimeTests.cs @@ -1,13 +1,13 @@ -using DeltaLake.Runtime; - -namespace DeltaLake.Tests.Runtime; - -public class RuntimeTests -{ - [Fact] - public void Create_Runtime_Test() - { - using var rt = new DeltaRuntime(new RuntimeOptions()); - Assert.NotNull(rt); - } +using DeltaLake.Runtime; + +namespace DeltaLake.Tests.Runtime; + +public class RuntimeTests +{ + [Fact] + public void Create_Runtime_Test() + { + using var rt = new DeltaRuntime(new RuntimeOptions()); + Assert.NotNull(rt); + } } \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Settings.cs b/tests/DeltaLake.Tests/Settings.cs index dd9aa5f..3625c93 100644 --- a/tests/DeltaLake.Tests/Settings.cs +++ b/tests/DeltaLake.Tests/Settings.cs @@ -1,7 +1,7 @@ -namespace DeltaLake.Tests -{ - public static class Settings - { - public static string TestRoot = System.Environment.GetEnvironmentVariable("TEST_ROOT") ?? Path.Join("..", "..", "..", "..", "data"); - } +namespace DeltaLake.Tests +{ + public static class Settings + { + public static string TestRoot = System.Environment.GetEnvironmentVariable("TEST_ROOT") ?? Path.Join("..", "..", "..", "..", "data"); + } } \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Table/AllTablesEnumerable.cs b/tests/DeltaLake.Tests/Table/AllTablesEnumerable.cs index 49bb28f..8eb37a5 100644 --- a/tests/DeltaLake.Tests/Table/AllTablesEnumerable.cs +++ b/tests/DeltaLake.Tests/Table/AllTablesEnumerable.cs @@ -1,14 +1,14 @@ -using System.Collections; - -namespace DeltaLake.Tests.Table; -public partial class LoadTests -{ - public class AllTablesEnumerable : IEnumerable - { - public IEnumerator GetEnumerator() => TableHelpers.ValidTables.Select(x => new object[] { - x - }).GetEnumerator(); - - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - } +using System.Collections; + +namespace DeltaLake.Tests.Table; +public partial class LoadTests +{ + public class AllTablesEnumerable : IEnumerable + { + public IEnumerator GetEnumerator() => TableHelpers.ValidTables.Select(x => new object[] { + x + }).GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } } \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Table/ConstraintTests.cs b/tests/DeltaLake.Tests/Table/ConstraintTests.cs index 4aa23ee..0b722f2 100644 --- a/tests/DeltaLake.Tests/Table/ConstraintTests.cs +++ b/tests/DeltaLake.Tests/Table/ConstraintTests.cs @@ -1,165 +1,165 @@ -using Apache.Arrow; -using Apache.Arrow.Memory; -using Apache.Arrow.Types; -using DeltaLake.Errors; -using DeltaLake.Runtime; -using DeltaLake.Table; - -namespace DeltaLake.Tests.Table -{ - public class ConstraintTests - { - [Theory] - [MemberData(nameof(TestCases))] - public async Task Contraint_Greater_Memory_Test( - int first, - string second, - long third, - Dictionary constraints, - bool throws - ) - { - Func task = () => BaseConstraintTest( - $"memory://{Guid.NewGuid():N}", - async table => - { - await table.AddConstraintsAsync( - constraints, - CancellationToken.None); - }, - first, - second, - third - ); - if (throws) - { - await Assert.ThrowsAsync(task); - } - else - { - await task(); - } - } - - [Fact] - public async Task Invalid_Constraint_Test() - { - var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); - using var runtime = tableParts.runtime; - using var table = tableParts.table; - await Assert.ThrowsAsync(() => table.AddConstraintsAsync( - new Dictionary - { - ["something isn't right"] = "invalid constraint", - }, - new Dictionary(), - CancellationToken.None)); - } - - [Fact] - public async Task Add_Constraint_Cancellation_Test() - { - var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); - using var runtime = tableParts.runtime; - using var table = tableParts.table; - var version = table.Version(); - await Assert.ThrowsAnyAsync(() => table.AddConstraintsAsync( - new Dictionary - { - ["something isn't right"] = "invalid constraint", - }, - new Dictionary(), - new CancellationToken(true))); - Assert.Equal(version, table.Version()); - } - - [Fact] - public async Task Empty_Constraint_Test() - { - var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); - using var runtime = tableParts.runtime; - using var table = tableParts.table; - var version = table.Version(); - await table.AddConstraintsAsync( - new Dictionary(), - new Dictionary(), - CancellationToken.None); - Assert.Equal(version, table.Version()); - } - - public static IEnumerable TestCases() - { - yield return [1, "hello", 0, new Dictionary { ["first"] = "first > 0" }, false]; - yield return [0, - "hello", - 0, - new Dictionary - { - ["first"] = "first > 0", - }, - true]; - yield return [0, - "hello", - 1, - new Dictionary - { - ["third"] = "third > 0", - }, - false]; - yield return [0, - "hello", - 0, - new Dictionary - { - ["third"] = "third > 0", - }, - true]; - } - - private async Task BaseConstraintTest( - string path, - Func addConstraints, - int first, - string second, - long third) - { - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - var builder = new Schema.Builder(); - builder.Field(fb => - { - fb.Name("first"); - fb.DataType(Int32Type.Default); - fb.Nullable(false); - }) - .Field(fb => - { - fb.Name("second"); - fb.DataType(StringType.Default); - fb.Nullable(false); - }) - .Field(fb => - { - fb.Name("third"); - fb.DataType(Int64Type.Default); - fb.Nullable(false); - }); - var schema = builder.Build(); - using var table = await DeltaTable.CreateAsync( - runtime, - new TableCreateOptions(path, schema), - CancellationToken.None); - await addConstraints(table); - var allocator = new NativeMemoryAllocator(); - var recordBatchBuilder = new RecordBatch.Builder(allocator) - .Append("test", false, col => col.Int32(arr => arr.Append(first))) - .Append("second", false, col => col.String(arr => arr.Append(second))) - .Append("third", false, col => col.Int64(arr => arr.Append(third))); - var options = new InsertOptions - { - SaveMode = SaveMode.Append, - }; - await table.InsertAsync([recordBatchBuilder.Build()], schema, options, CancellationToken.None); - - } - } +using Apache.Arrow; +using Apache.Arrow.Memory; +using Apache.Arrow.Types; +using DeltaLake.Errors; +using DeltaLake.Runtime; +using DeltaLake.Table; + +namespace DeltaLake.Tests.Table +{ + public class ConstraintTests + { + [Theory] + [MemberData(nameof(TestCases))] + public async Task Contraint_Greater_Memory_Test( + int first, + string second, + long third, + Dictionary constraints, + bool throws + ) + { + Func task = () => BaseConstraintTest( + $"memory://{Guid.NewGuid():N}", + async table => + { + await table.AddConstraintsAsync( + constraints, + CancellationToken.None); + }, + first, + second, + third + ); + if (throws) + { + await Assert.ThrowsAsync(task); + } + else + { + await task(); + } + } + + [Fact] + public async Task Invalid_Constraint_Test() + { + var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); + using var runtime = tableParts.runtime; + using var table = tableParts.table; + await Assert.ThrowsAsync(() => table.AddConstraintsAsync( + new Dictionary + { + ["something isn't right"] = "invalid constraint", + }, + new Dictionary(), + CancellationToken.None)); + } + + [Fact] + public async Task Add_Constraint_Cancellation_Test() + { + var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); + using var runtime = tableParts.runtime; + using var table = tableParts.table; + var version = table.Version(); + await Assert.ThrowsAnyAsync(() => table.AddConstraintsAsync( + new Dictionary + { + ["something isn't right"] = "invalid constraint", + }, + new Dictionary(), + new CancellationToken(true))); + Assert.Equal(version, table.Version()); + } + + [Fact] + public async Task Empty_Constraint_Test() + { + var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); + using var runtime = tableParts.runtime; + using var table = tableParts.table; + var version = table.Version(); + await table.AddConstraintsAsync( + new Dictionary(), + new Dictionary(), + CancellationToken.None); + Assert.Equal(version, table.Version()); + } + + public static IEnumerable TestCases() + { + yield return [1, "hello", 0, new Dictionary { ["first"] = "first > 0" }, false]; + yield return [0, + "hello", + 0, + new Dictionary + { + ["first"] = "first > 0", + }, + true]; + yield return [0, + "hello", + 1, + new Dictionary + { + ["third"] = "third > 0", + }, + false]; + yield return [0, + "hello", + 0, + new Dictionary + { + ["third"] = "third > 0", + }, + true]; + } + + private async Task BaseConstraintTest( + string path, + Func addConstraints, + int first, + string second, + long third) + { + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + var builder = new Schema.Builder(); + builder.Field(fb => + { + fb.Name("first"); + fb.DataType(Int32Type.Default); + fb.Nullable(false); + }) + .Field(fb => + { + fb.Name("second"); + fb.DataType(StringType.Default); + fb.Nullable(false); + }) + .Field(fb => + { + fb.Name("third"); + fb.DataType(Int64Type.Default); + fb.Nullable(false); + }); + var schema = builder.Build(); + using var table = await DeltaTable.CreateAsync( + runtime, + new TableCreateOptions(path, schema), + CancellationToken.None); + await addConstraints(table); + var allocator = new NativeMemoryAllocator(); + var recordBatchBuilder = new RecordBatch.Builder(allocator) + .Append("test", false, col => col.Int32(arr => arr.Append(first))) + .Append("second", false, col => col.String(arr => arr.Append(second))) + .Append("third", false, col => col.Int64(arr => arr.Append(third))); + var options = new InsertOptions + { + SaveMode = SaveMode.Append, + }; + await table.InsertAsync([recordBatchBuilder.Build()], schema, options, CancellationToken.None); + + } + } } \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Table/DeleteTests.cs b/tests/DeltaLake.Tests/Table/DeleteTests.cs index bed6a31..403ccae 100644 --- a/tests/DeltaLake.Tests/Table/DeleteTests.cs +++ b/tests/DeltaLake.Tests/Table/DeleteTests.cs @@ -1,114 +1,114 @@ -using DeltaLake.Errors; -using DeltaLake.Table; - -namespace DeltaLake.Tests.Table; -public class DeleteTests -{ - public static IEnumerable BaseCases() - { -#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. - yield return [1, default(string?), 0]; -#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. - yield return [1, "test < CAST(0 AS INT)", 1]; - yield return [1, "test >= CAST(0 AS INT)", 0]; - yield return [1, "second == 'test'", 1]; - yield return [1, "second == '0'", 0]; - yield return [1, "third < 1", 0]; -#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. - yield return [2, default(string?), 0]; -#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. - yield return [2, "test < CAST(0 AS INT)", 2]; - yield return [2, "test >= CAST(0 AS INT)", 0]; - yield return [2, "second == 'test'", 2]; - yield return [2, "second == '0'", 1]; - yield return [2, "third < 1", 1]; -#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. - yield return [10, default(string?), 0]; -#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. - yield return [10, "test < CAST(0 AS INT)", 10]; - yield return [10, "test >= CAST(0 AS INT)", 0]; - yield return [10, "second == 'test'", 10]; - yield return [10, "second == '0'", 9]; - yield return [10, "third < 1", 9]; -#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. - yield return [100, default(string?), 0]; -#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. - yield return [100, "test < CAST(0 AS INT)", 100]; - yield return [100, "test >= CAST(0 AS INT)", 0]; - yield return [100, "second == 'test'", 100]; - yield return [100, "second == '0'", 99]; - yield return [100, "third < 1", 99]; - } - - [Theory] - [MemberData(nameof(BaseCases))] - public async Task Memory_Insert_Variable_Record_Delete_Test( - int length, - string? predicate, - int expectedRecords) - { - await BaseDeleteTest($"memory://{Guid.NewGuid():N}", length, predicate, expectedRecords); - } - - [Theory] - [MemberData(nameof(BaseCases))] - public async Task File_System_Insert_Variable_Record_Delete_Test( - int length, - string? predicate, - int expectedRecords) - { - var info = Directory.CreateTempSubdirectory(); - try - { - await BaseDeleteTest($"file://{info.FullName}", length, predicate, expectedRecords); - } - finally - { - info.Delete(true); - } - } - - [Fact] - public async Task Memory_Invalid_Delete_Predicate_Test() - { - await Assert.ThrowsAsync(async () => - await BaseDeleteTest($"memory://{Guid.NewGuid():N}", 10, "invalid_property > 100", 10)); - } - - [Fact] - public async Task Memory_Delete_Cancellation_Test() - { - await Assert.ThrowsAnyAsync(async () => - await BaseDeleteTest($"memory://{Guid.NewGuid():N}", 10, default, 10, true)); - } - - private async Task BaseDeleteTest( - string path, - int length, - string? predicate, - int expectedRecords, - bool cancelOp = false) - { - var data = await TableHelpers.SetupTable(path, length); - using var runtime = data.runtime; - using var table = data.table; - var token = cancelOp ? new CancellationToken(true) : CancellationToken.None; - if (predicate == null) - { - await table.DeleteAsync(token); - } - else - { - await table.DeleteAsync(predicate, token); - } - - var queryResult = table.QueryAsync(new SelectQuery("select * from test") - { - TableAlias = "test", - }, - CancellationToken.None).ToBlockingEnumerable().ToList(); - - var totalRecords = queryResult.Select(s => s.Length).Sum(); - Assert.Equal(expectedRecords, totalRecords); - } +using DeltaLake.Errors; +using DeltaLake.Table; + +namespace DeltaLake.Tests.Table; +public class DeleteTests +{ + public static IEnumerable BaseCases() + { +#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. + yield return [1, default(string?), 0]; +#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + yield return [1, "test < CAST(0 AS INT)", 1]; + yield return [1, "test >= CAST(0 AS INT)", 0]; + yield return [1, "second == 'test'", 1]; + yield return [1, "second == '0'", 0]; + yield return [1, "third < 1", 0]; +#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. + yield return [2, default(string?), 0]; +#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + yield return [2, "test < CAST(0 AS INT)", 2]; + yield return [2, "test >= CAST(0 AS INT)", 0]; + yield return [2, "second == 'test'", 2]; + yield return [2, "second == '0'", 1]; + yield return [2, "third < 1", 1]; +#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. + yield return [10, default(string?), 0]; +#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + yield return [10, "test < CAST(0 AS INT)", 10]; + yield return [10, "test >= CAST(0 AS INT)", 0]; + yield return [10, "second == 'test'", 10]; + yield return [10, "second == '0'", 9]; + yield return [10, "third < 1", 9]; +#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. + yield return [100, default(string?), 0]; +#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + yield return [100, "test < CAST(0 AS INT)", 100]; + yield return [100, "test >= CAST(0 AS INT)", 0]; + yield return [100, "second == 'test'", 100]; + yield return [100, "second == '0'", 99]; + yield return [100, "third < 1", 99]; + } + + [Theory] + [MemberData(nameof(BaseCases))] + public async Task Memory_Insert_Variable_Record_Delete_Test( + int length, + string? predicate, + int expectedRecords) + { + await BaseDeleteTest($"memory://{Guid.NewGuid():N}", length, predicate, expectedRecords); + } + + [Theory] + [MemberData(nameof(BaseCases))] + public async Task File_System_Insert_Variable_Record_Delete_Test( + int length, + string? predicate, + int expectedRecords) + { + var info = Directory.CreateTempSubdirectory(); + try + { + await BaseDeleteTest($"file://{info.FullName}", length, predicate, expectedRecords); + } + finally + { + info.Delete(true); + } + } + + [Fact] + public async Task Memory_Invalid_Delete_Predicate_Test() + { + await Assert.ThrowsAsync(async () => + await BaseDeleteTest($"memory://{Guid.NewGuid():N}", 10, "invalid_property > 100", 10)); + } + + [Fact] + public async Task Memory_Delete_Cancellation_Test() + { + await Assert.ThrowsAnyAsync(async () => + await BaseDeleteTest($"memory://{Guid.NewGuid():N}", 10, default, 10, true)); + } + + private async Task BaseDeleteTest( + string path, + int length, + string? predicate, + int expectedRecords, + bool cancelOp = false) + { + var data = await TableHelpers.SetupTable(path, length); + using var runtime = data.runtime; + using var table = data.table; + var token = cancelOp ? new CancellationToken(true) : CancellationToken.None; + if (predicate == null) + { + await table.DeleteAsync(token); + } + else + { + await table.DeleteAsync(predicate, token); + } + + var queryResult = table.QueryAsync(new SelectQuery("select * from test") + { + TableAlias = "test", + }, + CancellationToken.None).ToBlockingEnumerable().ToList(); + + var totalRecords = queryResult.Select(s => s.Length).Sum(); + Assert.Equal(expectedRecords, totalRecords); + } } \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Table/InsertTests.cs b/tests/DeltaLake.Tests/Table/InsertTests.cs index f65a378..211ac7d 100644 --- a/tests/DeltaLake.Tests/Table/InsertTests.cs +++ b/tests/DeltaLake.Tests/Table/InsertTests.cs @@ -1,142 +1,142 @@ -using DeltaLake.Errors; -using DeltaLake.Table; - -namespace DeltaLake.Tests.Table; -public class InsertTests -{ - [Theory] - [InlineData(1)] - [InlineData(2)] - [InlineData(10)] - [InlineData(100)] - public async Task Memory_Insert_Variable_Record_Count_Test(int length) - { - await BaseInsertTest($"memory://{Guid.NewGuid():N}", length); - } - - [Theory] - [InlineData(1)] - [InlineData(2)] - [InlineData(10)] - [InlineData(100)] - public async Task File_System_Insert_Variable_Record_Count_Test(int length) - { - var info = Directory.CreateTempSubdirectory(); - try - { - await BaseInsertTest($"file://{info.FullName}", length); - } - finally - { - info.Delete(true); - } - } - - [Fact] - public async Task Memory_Insert_Zero_Record_Count_Test() - { - var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); - using var runtime = tableParts.runtime; - using var table = tableParts.table; - var version = table.Version(); - await table.InsertAsync([], table.Schema(), new InsertOptions(), CancellationToken.None); - Assert.Equal(version, table.Version()); - } - - [Fact] - public async Task Insert_Stream_Test() - { - var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); - using var runtime = tableParts.runtime; - using var table = tableParts.table; - var version = table.Version(); - var rb = new[] { - TableHelpers.BuildBasicRecordBatch(10), - }; - var schema = table.Schema(); - using var reader = new Bridge.RecordBatchReader(rb, schema); - await table.InsertAsync(reader, new InsertOptions(), CancellationToken.None); - Assert.Equal(version + 1, table.Version()); - } - - [Fact] - public async Task Memory_Insert_Will_Cancel_Test() - { - var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); - using var runtime = tableParts.runtime; - using var table = tableParts.table; - var version = table.Version(); - try - { - await table.InsertAsync([TableHelpers.BuildBasicRecordBatch(10)], table.Schema(), new InsertOptions(), new CancellationToken(true)); - throw new InvalidOperationException(); - } - catch (OperationCanceledException) - { - Assert.Equal(version, table.Version()); - } - } - - [Fact] - public async Task Memory_Insert_Invalid_Schema_Options_Test() - { - await Assert.ThrowsAsync(async () => - { - var options = new InsertOptions - { - SaveMode = SaveMode.Append, - OverwriteSchema = true, - }; - var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 1, options); - using var table = tableParts.table; - var version = table.Version(); - await table.InsertAsync([], table.Schema(), new InsertOptions(), CancellationToken.None); - Assert.Equal(version, table.Version()); - throw new System.NotImplementedException(); - }); - } - - private async Task BaseInsertTest(string path, int length) - { - var data = await TableHelpers.SetupTable(path, length); - using var runtime = data.runtime; - using var table = data.table; - var queryResult = table.QueryAsync(new SelectQuery("SELECT test FROM test WHERE test > 1") - { - TableAlias = "test", - }, - CancellationToken.None).ToBlockingEnumerable().ToList(); - - if (length > 2) - { - var totalRecords = queryResult.Select(s => s.Length).Sum(); - Assert.Equal(length - 2, totalRecords); - } - else - { - Assert.Empty(queryResult); - } - } - - private async Task StreamInsertTest(string path, int length) - { - var data = await TableHelpers.SetupTable(path, length); - using var runtime = data.runtime; - using var table = data.table; - var queryResult = table.QueryAsync(new SelectQuery("SELECT test FROM test WHERE test > 1") - { - TableAlias = "test", - }, - CancellationToken.None).ToBlockingEnumerable().ToList(); - - if (length > 2) - { - var totalRecords = queryResult.Select(s => s.Length).Sum(); - Assert.Equal(length - 2, totalRecords); - } - else - { - Assert.Empty(queryResult); - } - } +using DeltaLake.Errors; +using DeltaLake.Table; + +namespace DeltaLake.Tests.Table; +public class InsertTests +{ + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(10)] + [InlineData(100)] + public async Task Memory_Insert_Variable_Record_Count_Test(int length) + { + await BaseInsertTest($"memory://{Guid.NewGuid():N}", length); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(10)] + [InlineData(100)] + public async Task File_System_Insert_Variable_Record_Count_Test(int length) + { + var info = Directory.CreateTempSubdirectory(); + try + { + await BaseInsertTest($"file://{info.FullName}", length); + } + finally + { + info.Delete(true); + } + } + + [Fact] + public async Task Memory_Insert_Zero_Record_Count_Test() + { + var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); + using var runtime = tableParts.runtime; + using var table = tableParts.table; + var version = table.Version(); + await table.InsertAsync([], table.Schema(), new InsertOptions(), CancellationToken.None); + Assert.Equal(version, table.Version()); + } + + [Fact] + public async Task Insert_Stream_Test() + { + var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); + using var runtime = tableParts.runtime; + using var table = tableParts.table; + var version = table.Version(); + var rb = new[] { + TableHelpers.BuildBasicRecordBatch(10), + }; + var schema = table.Schema(); + using var reader = new Bridge.RecordBatchReader(rb, schema); + await table.InsertAsync(reader, new InsertOptions(), CancellationToken.None); + Assert.Equal(version + 1, table.Version()); + } + + [Fact] + public async Task Memory_Insert_Will_Cancel_Test() + { + var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); + using var runtime = tableParts.runtime; + using var table = tableParts.table; + var version = table.Version(); + try + { + await table.InsertAsync([TableHelpers.BuildBasicRecordBatch(10)], table.Schema(), new InsertOptions(), new CancellationToken(true)); + throw new InvalidOperationException(); + } + catch (OperationCanceledException) + { + Assert.Equal(version, table.Version()); + } + } + + [Fact] + public async Task Memory_Insert_Invalid_Schema_Options_Test() + { + await Assert.ThrowsAsync(async () => + { + var options = new InsertOptions + { + SaveMode = SaveMode.Append, + OverwriteSchema = true, + }; + var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 1, options); + using var table = tableParts.table; + var version = table.Version(); + await table.InsertAsync([], table.Schema(), new InsertOptions(), CancellationToken.None); + Assert.Equal(version, table.Version()); + throw new System.NotImplementedException(); + }); + } + + private async Task BaseInsertTest(string path, int length) + { + var data = await TableHelpers.SetupTable(path, length); + using var runtime = data.runtime; + using var table = data.table; + var queryResult = table.QueryAsync(new SelectQuery("SELECT test FROM test WHERE test > 1") + { + TableAlias = "test", + }, + CancellationToken.None).ToBlockingEnumerable().ToList(); + + if (length > 2) + { + var totalRecords = queryResult.Select(s => s.Length).Sum(); + Assert.Equal(length - 2, totalRecords); + } + else + { + Assert.Empty(queryResult); + } + } + + private async Task StreamInsertTest(string path, int length) + { + var data = await TableHelpers.SetupTable(path, length); + using var runtime = data.runtime; + using var table = data.table; + var queryResult = table.QueryAsync(new SelectQuery("SELECT test FROM test WHERE test > 1") + { + TableAlias = "test", + }, + CancellationToken.None).ToBlockingEnumerable().ToList(); + + if (length > 2) + { + var totalRecords = queryResult.Select(s => s.Length).Sum(); + Assert.Equal(length - 2, totalRecords); + } + else + { + Assert.Empty(queryResult); + } + } } \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Table/LoadTests.cs b/tests/DeltaLake.Tests/Table/LoadTests.cs index 91c8037..22d30e0 100644 --- a/tests/DeltaLake.Tests/Table/LoadTests.cs +++ b/tests/DeltaLake.Tests/Table/LoadTests.cs @@ -1,323 +1,323 @@ -using System.Runtime.InteropServices; -using DeltaLake.Errors; -using DeltaLake.Runtime; -using DeltaLake.Table; - -namespace DeltaLake.Tests.Table; -public partial class LoadTests -{ - public static TableIdentifier[] AllTables = TableHelpers.Tables.Keys.ToArray(); - - [Theory] - [InlineData(1)] - [InlineData(2)] - [InlineData(3)] - [InlineData(4)] - public async Task Load_Table_At_Version_Test(ulong version) - { - var location = TableIdentifier.SimpleTable.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions - { - Version = version, - }, - CancellationToken.None); - Assert.Equal(version, table.Version()); - } - - [Theory] - [InlineData(1)] - [InlineData(2)] - [InlineData(3)] - [InlineData(4)] - public async Task Table_Load_Version_Test(ulong version) - { - var location = TableIdentifier.SimpleTable.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), - CancellationToken.None); - Assert.Equal(4UL, table.Version()); - await table.LoadVersionAsync(version, CancellationToken.None); - Assert.Equal(version, table.Version()); - } - - [Fact] - public async Task Table_Load_Invalid_Version_Test() - { - var location = TableIdentifier.SimpleTable.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), - CancellationToken.None); - Assert.Equal(4UL, table.Version()); - await Assert.ThrowsAsync(async () => - await table.LoadVersionAsync(ulong.MaxValue, CancellationToken.None)); - // table is in an invalid state - Assert.NotEqual(4UL, table.Version()); - } - - [Fact] - public async Task Table_Load_Version_Cancellation_Test() - { - var location = TableIdentifier.SimpleTable.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), - CancellationToken.None); - Assert.Equal(4UL, table.Version()); - await Assert.ThrowsAnyAsync(async () => - await table.LoadVersionAsync(4, new CancellationToken(true))); - Assert.Equal(4UL, table.Version()); - } - - - [Fact] - public async Task Table_Load_At_Now() - { - var dateTimeOffset = DateTimeOffset.UtcNow; - var location = TableIdentifier.SimpleTable.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions - { - Version = 1UL, - }, - CancellationToken.None); - Assert.Equal(1UL, table.Version()); - await table.LoadDateTimeAsync(dateTimeOffset, CancellationToken.None); - Assert.Equal(4UL, table.Version()); - } - - [Fact] - public async Task Table_Load_DateTime_Max_Value_Timestamp() - { - var dateTimeOffset = DateTimeOffset.MaxValue; - var location = TableIdentifier.SimpleTable.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions - { - Version = 1, - }, - CancellationToken.None); - Assert.Equal(1UL, table.Version()); - await table.LoadDateTimeAsync(dateTimeOffset, CancellationToken.None); - Assert.Equal(4UL, table.Version()); - } - - [Theory] - [InlineData(long.MaxValue)] - [InlineData(long.MinValue)] - public async Task Table_Load_Invalid_Timestamp(long value) - { - await Assert.ThrowsAsync(async () => - { - var location = TableIdentifier.SimpleTable.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions - { - Version = 1, - }, - CancellationToken.None); - Assert.Equal(1UL, table.Version()); - await table.LoadDateTimeAsync(value, CancellationToken.None); - Assert.Equal(4UL, table.Version()); - }); - } - - [Fact] - public async Task Table_Cancellation_Timestamp_Test() - { - await Assert.ThrowsAnyAsync(async () => - { - var location = TableIdentifier.SimpleTable.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions - { - Version = 1, - }, - CancellationToken.None); - Assert.Equal(1UL, table.Version()); - await table.LoadDateTimeAsync(long.MaxValue, new CancellationToken(true)); - Assert.Equal(1UL, table.Version()); - }); - } - - [Fact] - public async Task Table_Cancellation_DateTimeOffset_Test() - { - await Assert.ThrowsAnyAsync(async () => - { - var location = TableIdentifier.SimpleTable.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions - { - Version = 1, - }, - CancellationToken.None); - Assert.Equal(1UL, table.Version()); - await table.LoadDateTimeAsync(DateTimeOffset.UtcNow, new CancellationToken(true)); - Assert.Equal(1UL, table.Version()); - }); - } - - [Theory] - [InlineData(TableIdentifier.Checkpoints, 1, 12)] - [InlineData(TableIdentifier.CheckpointsVacuumed, 5, 12)] - [InlineData(TableIdentifier.Delta020, 1, 3)] - public async Task Table_Load_Latest(TableIdentifier identifier, int minVersion, int expectedVersion) - { - var location = identifier.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions - { - Version = (ulong)minVersion, - }, - CancellationToken.None); - Assert.Equal((ulong)minVersion, table.Version()); - await table.LoadDateTimeAsync(DateTimeOffset.UtcNow, CancellationToken.None); - Assert.Equal((ulong)expectedVersion, table.Version()); - } - - [Fact] - public async Task Table_Load_Latest_Cancel() - { - await Assert.ThrowsAnyAsync(async () => - { - var location = TableIdentifier.Checkpoints.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync( - runtime, - location, - new TableOptions(), - new CancellationToken(true)); - }); - } - - [Theory] - [InlineData(TableIdentifier.Checkpoints, 1, 12)] - [InlineData(TableIdentifier.CheckpointsVacuumed, 5, 12)] - [InlineData(TableIdentifier.Delta020, 1, 3)] - public async Task Table_Load_Update_Incremental(TableIdentifier identifier, int minVersion, int expectedVersion) - { - var location = identifier.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions - { - Version = (ulong)minVersion, - }, - CancellationToken.None); - Assert.Equal((ulong)minVersion, table.Version()); - for (var version = minVersion; version <= expectedVersion; version++) - { - await table.UpdateIncrementalAsync(version, CancellationToken.None); - Assert.Equal((ulong)version, table.Version()); - } - } - - [Theory] - [InlineData(TableIdentifier.Checkpoints, 1, 12)] - [InlineData(TableIdentifier.CheckpointsVacuumed, 5, 12)] - [InlineData(TableIdentifier.Delta020, 1, 3)] - public async Task Table_Load_Update_Incremental_Cancellation(TableIdentifier identifier, int minVersion, int expectedVersion) - { - var location = identifier.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions - { - Version = (ulong)minVersion, - }, - CancellationToken.None); - Assert.Equal((ulong)minVersion, table.Version()); - for (var version = minVersion; version <= expectedVersion; version++) - { - await Assert.ThrowsAnyAsync(() => - table.UpdateIncrementalAsync(version, new CancellationToken(true))); - Assert.Equal((ulong)minVersion, table.Version()); - } - } - - [Theory] - [InlineData(TableIdentifier.Checkpoints, 1, 12)] - [InlineData(TableIdentifier.CheckpointsVacuumed, 5, 12)] - [InlineData(TableIdentifier.Delta020, 1, 3)] - public async Task Table_Load_Update_Incremental_Latest(TableIdentifier identifier, int minVersion, int expectedVersion) - { - var location = identifier.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions - { - Version = (ulong)minVersion, - }, - CancellationToken.None); - Assert.Equal((ulong)minVersion, table.Version()); - await table.UpdateIncrementalAsync(100, CancellationToken.None); - Assert.Equal((ulong)expectedVersion, table.Version()); - } - - [Fact] - public async Task Table_Load_Invalid_Uri_Type_Test() - { - await Assert.ThrowsAsync(async () => - { - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, "invalid://invalid.uri", new TableOptions - { - Version = 50, - }, - CancellationToken.None); - }); - } - - [Fact] - public async Task Table_Load_Invalid_Path_Test() - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - await Assert.ThrowsAsync(TestBodyAsync); - } - else - { - - await Assert.ThrowsAsync(TestBodyAsync); - } - - static async Task TestBodyAsync() - { - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, "file://invalid.uri", new TableOptions - { - Version = 50, - }, - CancellationToken.None); - } - } - - [Theory] - [ClassData(typeof(AllTablesEnumerable))] - public async Task Table_Will_Load_Test(TableIdentifier identifier) - { - var location = identifier.TablePath(); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), - CancellationToken.None); - switch (identifier) - { - case TableIdentifier.Covid19NYT: - case TableIdentifier.Delta220PartitionedTypes: - case TableIdentifier.SimpleCommit: - case TableIdentifier.Delta08NumericPartition: - case TableIdentifier.Delta08Date: - case TableIdentifier.Golden: - case TableIdentifier.ConcurrentWorkers: - case TableIdentifier.Delta08NullPartition: - case TableIdentifier.Delta08Partition: - case TableIdentifier.Delta08SpecialPartition: - case TableIdentifier.TableWithColumnMapping: - case TableIdentifier.TableWithEdgeTimestamps: - case TableIdentifier.TableWithLiquidClustering: - case TableIdentifier.TableWithoutDvSmall: - Assert.Equal(0UL, table.Version()); - break; - default: - Assert.NotEqual(0UL, table.Version()); - break; - } - } +using System.Runtime.InteropServices; +using DeltaLake.Errors; +using DeltaLake.Runtime; +using DeltaLake.Table; + +namespace DeltaLake.Tests.Table; +public partial class LoadTests +{ + public static TableIdentifier[] AllTables = TableHelpers.Tables.Keys.ToArray(); + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + public async Task Load_Table_At_Version_Test(ulong version) + { + var location = TableIdentifier.SimpleTable.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = version, + }, + CancellationToken.None); + Assert.Equal(version, table.Version()); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + public async Task Table_Load_Version_Test(ulong version) + { + var location = TableIdentifier.SimpleTable.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), + CancellationToken.None); + Assert.Equal(4UL, table.Version()); + await table.LoadVersionAsync(version, CancellationToken.None); + Assert.Equal(version, table.Version()); + } + + [Fact] + public async Task Table_Load_Invalid_Version_Test() + { + var location = TableIdentifier.SimpleTable.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), + CancellationToken.None); + Assert.Equal(4UL, table.Version()); + await Assert.ThrowsAsync(async () => + await table.LoadVersionAsync(ulong.MaxValue, CancellationToken.None)); + // table is in an invalid state + Assert.NotEqual(4UL, table.Version()); + } + + [Fact] + public async Task Table_Load_Version_Cancellation_Test() + { + var location = TableIdentifier.SimpleTable.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), + CancellationToken.None); + Assert.Equal(4UL, table.Version()); + await Assert.ThrowsAnyAsync(async () => + await table.LoadVersionAsync(4, new CancellationToken(true))); + Assert.Equal(4UL, table.Version()); + } + + + [Fact] + public async Task Table_Load_At_Now() + { + var dateTimeOffset = DateTimeOffset.UtcNow; + var location = TableIdentifier.SimpleTable.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = 1UL, + }, + CancellationToken.None); + Assert.Equal(1UL, table.Version()); + await table.LoadDateTimeAsync(dateTimeOffset, CancellationToken.None); + Assert.Equal(4UL, table.Version()); + } + + [Fact] + public async Task Table_Load_DateTime_Max_Value_Timestamp() + { + var dateTimeOffset = DateTimeOffset.MaxValue; + var location = TableIdentifier.SimpleTable.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = 1, + }, + CancellationToken.None); + Assert.Equal(1UL, table.Version()); + await table.LoadDateTimeAsync(dateTimeOffset, CancellationToken.None); + Assert.Equal(4UL, table.Version()); + } + + [Theory] + [InlineData(long.MaxValue)] + [InlineData(long.MinValue)] + public async Task Table_Load_Invalid_Timestamp(long value) + { + await Assert.ThrowsAsync(async () => + { + var location = TableIdentifier.SimpleTable.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = 1, + }, + CancellationToken.None); + Assert.Equal(1UL, table.Version()); + await table.LoadDateTimeAsync(value, CancellationToken.None); + Assert.Equal(4UL, table.Version()); + }); + } + + [Fact] + public async Task Table_Cancellation_Timestamp_Test() + { + await Assert.ThrowsAnyAsync(async () => + { + var location = TableIdentifier.SimpleTable.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = 1, + }, + CancellationToken.None); + Assert.Equal(1UL, table.Version()); + await table.LoadDateTimeAsync(long.MaxValue, new CancellationToken(true)); + Assert.Equal(1UL, table.Version()); + }); + } + + [Fact] + public async Task Table_Cancellation_DateTimeOffset_Test() + { + await Assert.ThrowsAnyAsync(async () => + { + var location = TableIdentifier.SimpleTable.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = 1, + }, + CancellationToken.None); + Assert.Equal(1UL, table.Version()); + await table.LoadDateTimeAsync(DateTimeOffset.UtcNow, new CancellationToken(true)); + Assert.Equal(1UL, table.Version()); + }); + } + + [Theory] + [InlineData(TableIdentifier.Checkpoints, 1, 12)] + [InlineData(TableIdentifier.CheckpointsVacuumed, 5, 12)] + [InlineData(TableIdentifier.Delta020, 1, 3)] + public async Task Table_Load_Latest(TableIdentifier identifier, int minVersion, int expectedVersion) + { + var location = identifier.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = (ulong)minVersion, + }, + CancellationToken.None); + Assert.Equal((ulong)minVersion, table.Version()); + await table.LoadDateTimeAsync(DateTimeOffset.UtcNow, CancellationToken.None); + Assert.Equal((ulong)expectedVersion, table.Version()); + } + + [Fact] + public async Task Table_Load_Latest_Cancel() + { + await Assert.ThrowsAnyAsync(async () => + { + var location = TableIdentifier.Checkpoints.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync( + runtime, + location, + new TableOptions(), + new CancellationToken(true)); + }); + } + + [Theory] + [InlineData(TableIdentifier.Checkpoints, 1, 12)] + [InlineData(TableIdentifier.CheckpointsVacuumed, 5, 12)] + [InlineData(TableIdentifier.Delta020, 1, 3)] + public async Task Table_Load_Update_Incremental(TableIdentifier identifier, int minVersion, int expectedVersion) + { + var location = identifier.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = (ulong)minVersion, + }, + CancellationToken.None); + Assert.Equal((ulong)minVersion, table.Version()); + for (var version = minVersion; version <= expectedVersion; version++) + { + await table.UpdateIncrementalAsync(version, CancellationToken.None); + Assert.Equal((ulong)version, table.Version()); + } + } + + [Theory] + [InlineData(TableIdentifier.Checkpoints, 1, 12)] + [InlineData(TableIdentifier.CheckpointsVacuumed, 5, 12)] + [InlineData(TableIdentifier.Delta020, 1, 3)] + public async Task Table_Load_Update_Incremental_Cancellation(TableIdentifier identifier, int minVersion, int expectedVersion) + { + var location = identifier.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = (ulong)minVersion, + }, + CancellationToken.None); + Assert.Equal((ulong)minVersion, table.Version()); + for (var version = minVersion; version <= expectedVersion; version++) + { + await Assert.ThrowsAnyAsync(() => + table.UpdateIncrementalAsync(version, new CancellationToken(true))); + Assert.Equal((ulong)minVersion, table.Version()); + } + } + + [Theory] + [InlineData(TableIdentifier.Checkpoints, 1, 12)] + [InlineData(TableIdentifier.CheckpointsVacuumed, 5, 12)] + [InlineData(TableIdentifier.Delta020, 1, 3)] + public async Task Table_Load_Update_Incremental_Latest(TableIdentifier identifier, int minVersion, int expectedVersion) + { + var location = identifier.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = (ulong)minVersion, + }, + CancellationToken.None); + Assert.Equal((ulong)minVersion, table.Version()); + await table.UpdateIncrementalAsync(100, CancellationToken.None); + Assert.Equal((ulong)expectedVersion, table.Version()); + } + + [Fact] + public async Task Table_Load_Invalid_Uri_Type_Test() + { + await Assert.ThrowsAsync(async () => + { + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, "invalid://invalid.uri", new TableOptions + { + Version = 50, + }, + CancellationToken.None); + }); + } + + [Fact] + public async Task Table_Load_Invalid_Path_Test() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + await Assert.ThrowsAsync(TestBodyAsync); + } + else + { + + await Assert.ThrowsAsync(TestBodyAsync); + } + + static async Task TestBodyAsync() + { + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, "file://invalid.uri", new TableOptions + { + Version = 50, + }, + CancellationToken.None); + } + } + + [Theory] + [ClassData(typeof(AllTablesEnumerable))] + public async Task Table_Will_Load_Test(TableIdentifier identifier) + { + var location = identifier.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), + CancellationToken.None); + switch (identifier) + { + case TableIdentifier.Covid19NYT: + case TableIdentifier.Delta220PartitionedTypes: + case TableIdentifier.SimpleCommit: + case TableIdentifier.Delta08NumericPartition: + case TableIdentifier.Delta08Date: + case TableIdentifier.Golden: + case TableIdentifier.ConcurrentWorkers: + case TableIdentifier.Delta08NullPartition: + case TableIdentifier.Delta08Partition: + case TableIdentifier.Delta08SpecialPartition: + case TableIdentifier.TableWithColumnMapping: + case TableIdentifier.TableWithEdgeTimestamps: + case TableIdentifier.TableWithLiquidClustering: + case TableIdentifier.TableWithoutDvSmall: + Assert.Equal(0UL, table.Version()); + break; + default: + Assert.NotEqual(0UL, table.Version()); + break; + } + } } \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Table/MergeTests.cs b/tests/DeltaLake.Tests/Table/MergeTests.cs index 38aa3a7..846a132 100644 --- a/tests/DeltaLake.Tests/Table/MergeTests.cs +++ b/tests/DeltaLake.Tests/Table/MergeTests.cs @@ -1,219 +1,219 @@ -using Apache.Arrow; -using Apache.Arrow.Memory; -using DeltaLake.Errors; -using DeltaLake.Table; - -namespace DeltaLake.Tests.Table; -public class MergeTests -{ - [Fact] - public async Task Merge_Memory_Full_Test() - { - var query = @"MERGE INTO mytable USING newdata - ON newdata.test = mytable.test - WHEN MATCHED THEN - UPDATE SET - second = newdata.second, - third = newdata.third - WHEN NOT MATCHED BY SOURCE THEN DELETE - WHEN NOT MATCHED BY TARGET - THEN INSERT ( - test, - second, - third - ) - VALUES ( - newdata.test, - 'inserted data', - 99 - )"; - await BaseMergeTest(query, batches => - { - var column1 = batches.SelectMany(batch => ((Int32Array)batch.Column(0)).Values.ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column1.SequenceEqual([5, 6, 7, 8, 9, 10, 11, 12, 13, 14])); - var column2 = batches.SelectMany(batch => ((IReadOnlyList)(StringArray)batch.Column(1)).ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column2.SequenceEqual(["hello", "hello", "hello", "hello", "hello", "inserted data", "inserted data", "inserted data", "inserted data", "inserted data"])); - var column3 = batches.SelectMany(batch => ((Int64Array)batch.Column(2)).Values.ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column3.SequenceEqual([99L, 99L, 99L, 99L, 99L, 100L, 100L, 100L, 100L, 100L])); - }); - } - - [Fact] - public async Task Merge_Memory_No_Delete_Test() - { - var query = @"MERGE INTO mytable USING newdata - ON newdata.test = mytable.test - WHEN MATCHED THEN - UPDATE SET - second = newdata.second, - third = newdata.third - WHEN NOT MATCHED BY TARGET - THEN INSERT ( - test, - second, - third - ) - VALUES ( - newdata.test, - 'inserted data', - 99 - )"; - await BaseMergeTest(query, batches => - { - var column1 = batches.SelectMany(batch => ((Int32Array)batch.Column(0)).Values.ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column1.SequenceEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])); - var column2 = batches.SelectMany(batch => ((IReadOnlyList)(StringArray)batch.Column(1)).ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column2.SequenceEqual(["0", "1", "2", "3", "4", "hello", "hello", "hello", "hello", "hello", "inserted data", "inserted data", "inserted data", "inserted data", "inserted data"])); - var column3 = batches.SelectMany(batch => ((Int64Array)batch.Column(2)).Values.ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column3.SequenceEqual([0L, 1L, 2L, 3L, 4L, 99L, 99L, 99L, 99L, 99L, 100L, 100L, 100L, 100L, 100L])); - }); - } - - [Fact] - public async Task Merge_Memory_No_Insert_Test() - { - var query = @"MERGE INTO mytable USING newdata - ON newdata.test = mytable.test - WHEN MATCHED THEN - UPDATE SET - second = newdata.second, - third = newdata.third"; - await BaseMergeTest(query, batches => - { - var column1 = batches.SelectMany(batch => ((Int32Array)batch.Column(0)).Values.ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column1.SequenceEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); - var column2 = batches.SelectMany(batch => ((IReadOnlyList)(StringArray)batch.Column(1)).ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column2.SequenceEqual(["0", "1", "2", "3", "4", "hello", "hello", "hello", "hello", "hello"])); - var column3 = batches.SelectMany(batch => ((Int64Array)batch.Column(2)).Values.ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column3.SequenceEqual([0L, 1L, 2L, 3L, 4L, 100L, 100L, 100L, 100L, 100L])); - }); - } - - [Fact] - public async Task Merge_Memory_No_Update_Test() - { - var query = @"MERGE INTO mytable USING newdata - ON newdata.test = mytable.test - WHEN NOT MATCHED BY TARGET - THEN INSERT ( - test, - second, - third - ) - VALUES ( - newdata.test, - 'inserted data', - 99 - )"; - await BaseMergeTest(query, batches => - { - var column1 = batches.SelectMany(batch => ((Int32Array)batch.Column(0)).Values.ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column1.SequenceEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])); - var column2 = batches.SelectMany(batch => ((IReadOnlyList)(StringArray)batch.Column(1)).ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column2.SequenceEqual(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "inserted data", "inserted data", "inserted data", "inserted data", "inserted data"])); - var column3 = batches.SelectMany(batch => ((Int64Array)batch.Column(2)).Values.ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column3.SequenceEqual([0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 99L, 99L, 99L, 99L, 99L])); - }); - } - - [Fact] - public async Task Merge_Memory_Invalid_Query_Test() - { - await Assert.ThrowsAsync(() => - { - var query = @"MERGE mytable USING newdata - ON newdata.test = mytable.test - WHEN MATCHED THEN - UPDATE SET - second = newdata.second, - third = newdata.third"; - return BaseMergeTest(query, batches => - { - var column1 = batches.SelectMany(batch => ((Int32Array)batch.Column(0)).Values.ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column1.SequenceEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); - var column2 = batches.SelectMany(batch => ((IReadOnlyList)(StringArray)batch.Column(1)).ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column2.SequenceEqual(["0", "1", "2", "3", "4", "hello", "hello", "hello", "hello", "hello"])); - var column3 = batches.SelectMany(batch => ((Int64Array)batch.Column(2)).Values.ToArray()).OrderBy(i => i).ToArray(); - Assert.True(column3.SequenceEqual([0L, 1L, 2L, 3L, 4L, 100L, 100L, 100L, 100L, 100L])); - }); - }); - } - - [Fact] - public async Task Merge_Memory_Cancel_Test() - { - var query = @"MERGE INTO mytable USING newdata - ON newdata.test = mytable.test - WHEN NOT MATCHED BY TARGET - THEN INSERT ( - test, - second, - third - ) - VALUES ( - newdata.test, - 'inserted data', - 99 - )"; - var pair = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 10); - using var runtime = pair.runtime; - using var table = pair.table; - var allocator = new NativeMemoryAllocator(); - var enumerable = Enumerable.Range(5, 10); - var recordBatchBuilder = new RecordBatch.Builder(allocator) - .Append("test", false, col => col.Int32(arr => arr.AppendRange(enumerable))) - .Append("second", false, col => col.String(arr => arr.AppendRange(enumerable.Select(_ => "hello")))) - .Append("third", false, col => col.Int64(arr => arr.AppendRange(enumerable.Select(_ => 100L)))); - using var rb = recordBatchBuilder.Build(); - var version = table.Version(); - try - { - await table.MergeAsync(query, [rb], rb.Schema, new CancellationToken(true)); - throw new InvalidOperationException(); - } - catch (OperationCanceledException) - { - Assert.Equal(version, table.Version()); - } - } - - [Fact] - public async Task Merge_Zero_Record_Count_Test() - { - var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); - using var runtime = tableParts.runtime; - using var table = tableParts.table; - var version = table.Version(); - await table.MergeAsync(@"MERGE INTO mytable USING newdata - ON newdata.test = mytable.test - WHEN NOT MATCHED BY TARGET - THEN INSERT ( - test, - second, - third - ) - VALUES ( - newdata.test, - 'inserted data', - 99 - )", [], table.Schema(), CancellationToken.None); - Assert.Equal(version, table.Version()); - } - - private async Task BaseMergeTest(string query, Action> assertions) - { - var pair = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 10); - using var runtime = pair.runtime; - using var table = pair.table; - var allocator = new NativeMemoryAllocator(); - var enumerable = Enumerable.Range(5, 10); - var recordBatchBuilder = new RecordBatch.Builder(allocator) - .Append("test", false, col => col.Int32(arr => arr.AppendRange(enumerable))) - .Append("second", false, col => col.String(arr => arr.AppendRange(enumerable.Select(_ => "hello")))) - .Append("third", false, col => col.Int64(arr => arr.AppendRange(enumerable.Select(_ => 100L)))); - using var rb = recordBatchBuilder.Build(); - await table.MergeAsync(query, [rb], rb.Schema, CancellationToken.None); - var batches = table.QueryAsync(new SelectQuery("select * from deltatable"), CancellationToken.None).ToBlockingEnumerable().ToList(); - assertions(batches); - } +using Apache.Arrow; +using Apache.Arrow.Memory; +using DeltaLake.Errors; +using DeltaLake.Table; + +namespace DeltaLake.Tests.Table; +public class MergeTests +{ + [Fact] + public async Task Merge_Memory_Full_Test() + { + var query = @"MERGE INTO mytable USING newdata + ON newdata.test = mytable.test + WHEN MATCHED THEN + UPDATE SET + second = newdata.second, + third = newdata.third + WHEN NOT MATCHED BY SOURCE THEN DELETE + WHEN NOT MATCHED BY TARGET + THEN INSERT ( + test, + second, + third + ) + VALUES ( + newdata.test, + 'inserted data', + 99 + )"; + await BaseMergeTest(query, batches => + { + var column1 = batches.SelectMany(batch => ((Int32Array)batch.Column(0)).Values.ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column1.SequenceEqual([5, 6, 7, 8, 9, 10, 11, 12, 13, 14])); + var column2 = batches.SelectMany(batch => ((IReadOnlyList)(StringArray)batch.Column(1)).ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column2.SequenceEqual(["hello", "hello", "hello", "hello", "hello", "inserted data", "inserted data", "inserted data", "inserted data", "inserted data"])); + var column3 = batches.SelectMany(batch => ((Int64Array)batch.Column(2)).Values.ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column3.SequenceEqual([99L, 99L, 99L, 99L, 99L, 100L, 100L, 100L, 100L, 100L])); + }); + } + + [Fact] + public async Task Merge_Memory_No_Delete_Test() + { + var query = @"MERGE INTO mytable USING newdata + ON newdata.test = mytable.test + WHEN MATCHED THEN + UPDATE SET + second = newdata.second, + third = newdata.third + WHEN NOT MATCHED BY TARGET + THEN INSERT ( + test, + second, + third + ) + VALUES ( + newdata.test, + 'inserted data', + 99 + )"; + await BaseMergeTest(query, batches => + { + var column1 = batches.SelectMany(batch => ((Int32Array)batch.Column(0)).Values.ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column1.SequenceEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])); + var column2 = batches.SelectMany(batch => ((IReadOnlyList)(StringArray)batch.Column(1)).ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column2.SequenceEqual(["0", "1", "2", "3", "4", "hello", "hello", "hello", "hello", "hello", "inserted data", "inserted data", "inserted data", "inserted data", "inserted data"])); + var column3 = batches.SelectMany(batch => ((Int64Array)batch.Column(2)).Values.ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column3.SequenceEqual([0L, 1L, 2L, 3L, 4L, 99L, 99L, 99L, 99L, 99L, 100L, 100L, 100L, 100L, 100L])); + }); + } + + [Fact] + public async Task Merge_Memory_No_Insert_Test() + { + var query = @"MERGE INTO mytable USING newdata + ON newdata.test = mytable.test + WHEN MATCHED THEN + UPDATE SET + second = newdata.second, + third = newdata.third"; + await BaseMergeTest(query, batches => + { + var column1 = batches.SelectMany(batch => ((Int32Array)batch.Column(0)).Values.ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column1.SequenceEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); + var column2 = batches.SelectMany(batch => ((IReadOnlyList)(StringArray)batch.Column(1)).ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column2.SequenceEqual(["0", "1", "2", "3", "4", "hello", "hello", "hello", "hello", "hello"])); + var column3 = batches.SelectMany(batch => ((Int64Array)batch.Column(2)).Values.ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column3.SequenceEqual([0L, 1L, 2L, 3L, 4L, 100L, 100L, 100L, 100L, 100L])); + }); + } + + [Fact] + public async Task Merge_Memory_No_Update_Test() + { + var query = @"MERGE INTO mytable USING newdata + ON newdata.test = mytable.test + WHEN NOT MATCHED BY TARGET + THEN INSERT ( + test, + second, + third + ) + VALUES ( + newdata.test, + 'inserted data', + 99 + )"; + await BaseMergeTest(query, batches => + { + var column1 = batches.SelectMany(batch => ((Int32Array)batch.Column(0)).Values.ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column1.SequenceEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])); + var column2 = batches.SelectMany(batch => ((IReadOnlyList)(StringArray)batch.Column(1)).ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column2.SequenceEqual(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "inserted data", "inserted data", "inserted data", "inserted data", "inserted data"])); + var column3 = batches.SelectMany(batch => ((Int64Array)batch.Column(2)).Values.ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column3.SequenceEqual([0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 99L, 99L, 99L, 99L, 99L])); + }); + } + + [Fact] + public async Task Merge_Memory_Invalid_Query_Test() + { + await Assert.ThrowsAsync(() => + { + var query = @"MERGE mytable USING newdata + ON newdata.test = mytable.test + WHEN MATCHED THEN + UPDATE SET + second = newdata.second, + third = newdata.third"; + return BaseMergeTest(query, batches => + { + var column1 = batches.SelectMany(batch => ((Int32Array)batch.Column(0)).Values.ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column1.SequenceEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); + var column2 = batches.SelectMany(batch => ((IReadOnlyList)(StringArray)batch.Column(1)).ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column2.SequenceEqual(["0", "1", "2", "3", "4", "hello", "hello", "hello", "hello", "hello"])); + var column3 = batches.SelectMany(batch => ((Int64Array)batch.Column(2)).Values.ToArray()).OrderBy(i => i).ToArray(); + Assert.True(column3.SequenceEqual([0L, 1L, 2L, 3L, 4L, 100L, 100L, 100L, 100L, 100L])); + }); + }); + } + + [Fact] + public async Task Merge_Memory_Cancel_Test() + { + var query = @"MERGE INTO mytable USING newdata + ON newdata.test = mytable.test + WHEN NOT MATCHED BY TARGET + THEN INSERT ( + test, + second, + third + ) + VALUES ( + newdata.test, + 'inserted data', + 99 + )"; + var pair = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 10); + using var runtime = pair.runtime; + using var table = pair.table; + var allocator = new NativeMemoryAllocator(); + var enumerable = Enumerable.Range(5, 10); + var recordBatchBuilder = new RecordBatch.Builder(allocator) + .Append("test", false, col => col.Int32(arr => arr.AppendRange(enumerable))) + .Append("second", false, col => col.String(arr => arr.AppendRange(enumerable.Select(_ => "hello")))) + .Append("third", false, col => col.Int64(arr => arr.AppendRange(enumerable.Select(_ => 100L)))); + using var rb = recordBatchBuilder.Build(); + var version = table.Version(); + try + { + await table.MergeAsync(query, [rb], rb.Schema, new CancellationToken(true)); + throw new InvalidOperationException(); + } + catch (OperationCanceledException) + { + Assert.Equal(version, table.Version()); + } + } + + [Fact] + public async Task Merge_Zero_Record_Count_Test() + { + var tableParts = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 0); + using var runtime = tableParts.runtime; + using var table = tableParts.table; + var version = table.Version(); + await table.MergeAsync(@"MERGE INTO mytable USING newdata + ON newdata.test = mytable.test + WHEN NOT MATCHED BY TARGET + THEN INSERT ( + test, + second, + third + ) + VALUES ( + newdata.test, + 'inserted data', + 99 + )", [], table.Schema(), CancellationToken.None); + Assert.Equal(version, table.Version()); + } + + private async Task BaseMergeTest(string query, Action> assertions) + { + var pair = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 10); + using var runtime = pair.runtime; + using var table = pair.table; + var allocator = new NativeMemoryAllocator(); + var enumerable = Enumerable.Range(5, 10); + var recordBatchBuilder = new RecordBatch.Builder(allocator) + .Append("test", false, col => col.Int32(arr => arr.AppendRange(enumerable))) + .Append("second", false, col => col.String(arr => arr.AppendRange(enumerable.Select(_ => "hello")))) + .Append("third", false, col => col.Int64(arr => arr.AppendRange(enumerable.Select(_ => 100L)))); + using var rb = recordBatchBuilder.Build(); + await table.MergeAsync(query, [rb], rb.Schema, CancellationToken.None); + var batches = table.QueryAsync(new SelectQuery("select * from deltatable"), CancellationToken.None).ToBlockingEnumerable().ToList(); + assertions(batches); + } } \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Table/QueryTests.cs b/tests/DeltaLake.Tests/Table/QueryTests.cs index 2973c91..b374994 100644 --- a/tests/DeltaLake.Tests/Table/QueryTests.cs +++ b/tests/DeltaLake.Tests/Table/QueryTests.cs @@ -1,66 +1,66 @@ -using DeltaLake.Errors; -using DeltaLake.Table; - -namespace DeltaLake.Tests.Table; -public class QueryTests -{ - [Fact] - public async Task Query_Cancellation_Test() - { - var data = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 1); - using var runtime = data.runtime; - using var table = data.table; - await Assert.ThrowsAnyAsync(async () => - { - var query = new SelectQuery("select * from test") - { - TableAlias = "test", - }; - table.QueryAsync(query, new CancellationToken(true)) - .ToBlockingEnumerable() - .ToList(); - }); - } - - [Theory] - [InlineData("SELECT * FROM TEST", "test")] - [InlineData("SELECT * FROM test", "test")] - [InlineData("SELECT * FROM mytable", "mytable")] - public async Task Query_Valid_Query_Params_Test(string queryText, string tableAlias) - { - var data = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 1); - using var runtime = data.runtime; - using var table = data.table; - var query = new SelectQuery(queryText) - { - TableAlias = tableAlias, - }; - var results = table.QueryAsync(query, CancellationToken.None) - .ToBlockingEnumerable() - .ToList(); - Assert.Single(results); - Assert.Equal(1, results[0].Length); - } - - [Theory] - [InlineData("SELECT * FROM test", "junk")] - [InlineData("DELETE FROM TEST", "test")] - [InlineData("CREATE TABLE foo (x INTEGER)", "foo")] - [InlineData("UPDATE junk set first = 0", "junk")] - public async Task Query_Invalid_Query_Params_Test(string queryText, string tableAlias) - { - var data = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 1); - using var runtime = data.runtime; - using var table = data.table; - await Assert.ThrowsAsync(async () => - { - var query = new SelectQuery(queryText) - { - TableAlias = tableAlias, - }; - table.QueryAsync(query, CancellationToken.None) - .ToBlockingEnumerable() - .ToList(); - }); - } +using DeltaLake.Errors; +using DeltaLake.Table; + +namespace DeltaLake.Tests.Table; +public class QueryTests +{ + [Fact] + public async Task Query_Cancellation_Test() + { + var data = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 1); + using var runtime = data.runtime; + using var table = data.table; + await Assert.ThrowsAnyAsync(async () => + { + var query = new SelectQuery("select * from test") + { + TableAlias = "test", + }; + table.QueryAsync(query, new CancellationToken(true)) + .ToBlockingEnumerable() + .ToList(); + }); + } + + [Theory] + [InlineData("SELECT * FROM TEST", "test")] + [InlineData("SELECT * FROM test", "test")] + [InlineData("SELECT * FROM mytable", "mytable")] + public async Task Query_Valid_Query_Params_Test(string queryText, string tableAlias) + { + var data = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 1); + using var runtime = data.runtime; + using var table = data.table; + var query = new SelectQuery(queryText) + { + TableAlias = tableAlias, + }; + var results = table.QueryAsync(query, CancellationToken.None) + .ToBlockingEnumerable() + .ToList(); + Assert.Single(results); + Assert.Equal(1, results[0].Length); + } + + [Theory] + [InlineData("SELECT * FROM test", "junk")] + [InlineData("DELETE FROM TEST", "test")] + [InlineData("CREATE TABLE foo (x INTEGER)", "foo")] + [InlineData("UPDATE junk set first = 0", "junk")] + public async Task Query_Invalid_Query_Params_Test(string queryText, string tableAlias) + { + var data = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 1); + using var runtime = data.runtime; + using var table = data.table; + await Assert.ThrowsAsync(async () => + { + var query = new SelectQuery(queryText) + { + TableAlias = tableAlias, + }; + table.QueryAsync(query, CancellationToken.None) + .ToBlockingEnumerable() + .ToList(); + }); + } } \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Table/RestoreTests.cs b/tests/DeltaLake.Tests/Table/RestoreTests.cs index 3ddb789..f2c9c02 100644 --- a/tests/DeltaLake.Tests/Table/RestoreTests.cs +++ b/tests/DeltaLake.Tests/Table/RestoreTests.cs @@ -1,33 +1,33 @@ -using DeltaLake.Table; - -namespace DeltaLake.Tests.Table; - -public class RestoreTests -{ - [Theory] - [InlineData(true)] - [InlineData(false)] - public async Task Restore_Cancellation_Test(bool useVersion) - { - var data = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 1); - using var runtime = data.runtime; - using var table = data.table; - var options = new RestoreOptions - { - IgnoreMissingFiles = true, - ProtocolDowngradeAllowed = true, - Version = useVersion ? 1 : null, - Timestamp = useVersion ? null : DateTimeOffset.UtcNow, - }; - var version = table.Version(); - try - { - await table.RestoreAsync(options, new CancellationToken(true)); - throw new InvalidOperationException(); - } - catch (OperationCanceledException) - { - Assert.Equal(version, table.Version()); - } - } -} +using DeltaLake.Table; + +namespace DeltaLake.Tests.Table; + +public class RestoreTests +{ + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Restore_Cancellation_Test(bool useVersion) + { + var data = await TableHelpers.SetupTable($"memory://{Guid.NewGuid():N}", 1); + using var runtime = data.runtime; + using var table = data.table; + var options = new RestoreOptions + { + IgnoreMissingFiles = true, + ProtocolDowngradeAllowed = true, + Version = useVersion ? 1 : null, + Timestamp = useVersion ? null : DateTimeOffset.UtcNow, + }; + var version = table.Version(); + try + { + await table.RestoreAsync(options, new CancellationToken(true)); + throw new InvalidOperationException(); + } + catch (OperationCanceledException) + { + Assert.Equal(version, table.Version()); + } + } +} diff --git a/tests/DeltaLake.Tests/Usings.cs b/tests/DeltaLake.Tests/Usings.cs index c802f44..e106559 100644 --- a/tests/DeltaLake.Tests/Usings.cs +++ b/tests/DeltaLake.Tests/Usings.cs @@ -1 +1 @@ -global using Xunit; +global using Xunit; From 4dc05f747794b9049103984118263db6a86c1876 Mon Sep 17 00:00:00 2001 From: KyJah Keys Date: Fri, 27 Sep 2024 15:23:46 -0400 Subject: [PATCH 07/12] increased stack size for tests --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 60efdc3..fe0074d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,6 +64,8 @@ jobs: - name: Test env: RUST_BACKTRACE: 1 + # increased stack size currently necessary for osx + DOTNET_DefaultStackSize: "180000" run: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=lcov /p:CoverletOutput=./lcov.info --logger "console;verbosity=detailed" --blame-crash -v n - name: Upload test failure From fef442085df169ceaeeb314bf176c1f86a9eb45e Mon Sep 17 00:00:00 2001 From: KyJah Keys Date: Fri, 27 Sep 2024 15:36:53 -0400 Subject: [PATCH 08/12] fixed lints and added some documentation --- README.md | 1 + src/DeltaLake/Bridge/Runtime.cs | 14 +++++++------- src/DeltaLake/Bridge/Table.cs | 22 +++++++++++----------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 12bf458..6b078e2 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ This library also takes advantage of the [Apache Arrow](https://github.com/apach The bridge library incorporates delta-rs and [tokio-rs](https://tokio.rs/) as shown in the image below. ![alt text](/media/images/bridge-library.png "Rust bridge library with tokio") +NOTE: On unix systems, there is the possibility of a stack overflow due to small stack sizes for the .NET framework. The default size should correspond to `ulimit -s`, but we can override this by setting the environment variable `DOTNET_DefaultStackSize` to a hexadecimal number of bytes. The unit tests use `180000`. ## Quick Start ```csharp diff --git a/src/DeltaLake/Bridge/Runtime.cs b/src/DeltaLake/Bridge/Runtime.cs index 708f9f6..6225e3d 100644 --- a/src/DeltaLake/Bridge/Runtime.cs +++ b/src/DeltaLake/Bridge/Runtime.cs @@ -87,17 +87,17 @@ internal async Task
LoadTableAsync( { if (cancellationToken.IsCancellationRequested) { - Task.Run(() => tsc.TrySetCanceled(cancellationToken)); ; + _ = Task.Run(() => tsc.TrySetCanceled(cancellationToken)); ; return; } if (fail != null) { - Task.Run(() => tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(Ptr, fail))); + _ = Task.Run(() => tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(Ptr, fail))); } else { - Task.Run(() => tsc.TrySetResult(new Table(this, success))); + _ = Task.Run(() => tsc.TrySetResult(new Table(this, success))); } })); } @@ -127,7 +127,7 @@ internal async Task
CreateTableAsync(DeltaLake.Table.TableCreateOptions o mode = saveMode.Ref, name = scope.ByteArray(options.Name), description = scope.ByteArray(options.Description), - configuration = scope.OptionalDictionary(this, options.Configuration ?? new Dictionary()), + configuration = scope.Dictionary(this, options.Configuration ?? new Dictionary()), custom_metadata = scope.Dictionary(this, options.CustomMetadata ?? new Dictionary()), storage_options = scope.Dictionary(this, options.StorageOptions ?? new Dictionary()), }; @@ -139,17 +139,17 @@ internal async Task
CreateTableAsync(DeltaLake.Table.TableCreateOptions o { if (cancellationToken.IsCancellationRequested) { - Task.Run(() => tsc.TrySetCanceled(cancellationToken)); ; + _ = Task.Run(() => tsc.TrySetCanceled(cancellationToken)); ; return; } if (fail != null) { - Task.Run(() => tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(Ptr, fail))); + _ = Task.Run(() => tsc.TrySetException(DeltaRuntimeException.FromDeltaTableError(Ptr, fail))); } else { - Task.Run(() => tsc.TrySetResult(new Table(this, success))); + _ = Task.Run(() => tsc.TrySetResult(new Table(this, success))); } })); } diff --git a/src/DeltaLake/Bridge/Table.cs b/src/DeltaLake/Bridge/Table.cs index 34749c8..885e6a2 100644 --- a/src/DeltaLake/Bridge/Table.cs +++ b/src/DeltaLake/Bridge/Table.cs @@ -73,7 +73,7 @@ public async Task LoadVersionAsync(ulong version, ICancellationToken cancellatio } else { - Task.Run(() => tsc.TrySetResult(true)); + _ = Task.Run(() => tsc.TrySetResult(true)); } })); } @@ -108,7 +108,7 @@ internal async Task LoadTimestampAsync(long timestampMilliseconds, ICancellation } else { - Task.Run(() => tsc.TrySetResult(true)); + _ = Task.Run(() => tsc.TrySetResult(true)); } })); } @@ -258,7 +258,7 @@ public async Task InsertAsync( } else { - Task.Run(() => tsc.TrySetResult("{}")); + _ = Task.Run(() => tsc.TrySetResult("{}")); } })); } @@ -315,7 +315,7 @@ public async Task MergeAsync( { using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); var value = content.ToUTF8(); - Task.Run(() => tsc.TrySetResult(value)); + _ = Task.Run(() => tsc.TrySetResult(value)); } })); @@ -362,7 +362,7 @@ public async Task QueryAsync( else { var stream = CArrowArrayStreamImporter.ImportArrayStream((CArrowArrayStream*)success); - Task.Run(() => + _ = Task.Run(() => { if (!tsc.TrySetResult(stream)) { @@ -403,7 +403,7 @@ public async Task DeleteAsync(string predicate, ICancellationToken cance { using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); var value = content.ToUTF8(); - Task.Run(() => tsc.TrySetResult(value)); + _ = Task.Run(() => tsc.TrySetResult(value)); } })); @@ -439,7 +439,7 @@ public async Task UpdateAsync(string query, ICancellationToken cancellat { using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); var result = content.ToUTF8(); - Task.Run(() => tsc.TrySetResult(result)); + _ = Task.Run(() => tsc.TrySetResult(result)); } })); @@ -475,7 +475,7 @@ public async Task HistoryAsync(ulong limit, ICancellationToken cancellat { using var content = new ByteArray(_runtime, (Interop.ByteArray*)success); var bytes = content.ToByteArray(); - Task.Run(() => tsc.TrySetResult(bytes)); ; + _ = Task.Run(() => tsc.TrySetResult(bytes)); ; } })); @@ -515,7 +515,7 @@ public async Task AddConstraintAsync(IReadOnlyDictionary constra } else { - Task.Run(() => tsc.TrySetResult(true)); + _ = Task.Run(() => tsc.TrySetResult(true)); } })); } @@ -548,7 +548,7 @@ public async Task UpdateIncrementalAsync(long? maxVersion, ICancellationToken ca } else { - Task.Run(() => tsc.TrySetResult(true)); + _ = Task.Run(() => tsc.TrySetResult(true)); } })); @@ -609,7 +609,7 @@ public async Task RestoreAsync(RestoreOptions options, ICancellationToken cancel } else { - Task.Run(() => tsc.TrySetResult(true)); + _ = Task.Run(() => tsc.TrySetResult(true)); } })); } From dda8c896385c6de40c3817c627763b486431d016 Mon Sep 17 00:00:00 2001 From: KyJah Keys Date: Mon, 30 Sep 2024 09:57:06 -0400 Subject: [PATCH 09/12] changed tests to run against a release build --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe0074d..59aa1fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,7 @@ jobs: RUST_BACKTRACE: 1 # increased stack size currently necessary for osx DOTNET_DefaultStackSize: "180000" - run: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=lcov /p:CoverletOutput=./lcov.info --logger "console;verbosity=detailed" --blame-crash -v n + run: dotnet test -C Release /p:CollectCoverage=true /p:CoverletOutputFormat=lcov /p:CoverletOutput=./lcov.info --logger "console;verbosity=detailed" --blame-crash -v n - name: Upload test failure if: ${{ failure() }} From 3991fbcc81ceaea02072dd2fbcc2c03c5939162d Mon Sep 17 00:00:00 2001 From: KyJah Keys Date: Mon, 30 Sep 2024 10:03:06 -0400 Subject: [PATCH 10/12] corrected casing on test arg --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59aa1fd..d905953 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,7 @@ jobs: RUST_BACKTRACE: 1 # increased stack size currently necessary for osx DOTNET_DefaultStackSize: "180000" - run: dotnet test -C Release /p:CollectCoverage=true /p:CoverletOutputFormat=lcov /p:CoverletOutput=./lcov.info --logger "console;verbosity=detailed" --blame-crash -v n + run: dotnet test -c Release /p:CollectCoverage=true /p:CoverletOutputFormat=lcov /p:CoverletOutput=./lcov.info --logger "console;verbosity=detailed" --blame-crash -v n - name: Upload test failure if: ${{ failure() }} From 5af6f4814dbcd634c89567dc7f898470770ef223 Mon Sep 17 00:00:00 2001 From: KyJah Keys Date: Mon, 30 Sep 2024 10:33:28 -0400 Subject: [PATCH 11/12] removed windows-specified exception test --- tests/DeltaLake.Tests/Table/LoadTests.cs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/tests/DeltaLake.Tests/Table/LoadTests.cs b/tests/DeltaLake.Tests/Table/LoadTests.cs index 22d30e0..fa5718b 100644 --- a/tests/DeltaLake.Tests/Table/LoadTests.cs +++ b/tests/DeltaLake.Tests/Table/LoadTests.cs @@ -268,16 +268,7 @@ await Assert.ThrowsAsync(async () => [Fact] public async Task Table_Load_Invalid_Path_Test() { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - await Assert.ThrowsAsync(TestBodyAsync); - } - else - { - - await Assert.ThrowsAsync(TestBodyAsync); - } - + await Assert.ThrowsAsync(TestBodyAsync); static async Task TestBodyAsync() { using var runtime = new DeltaRuntime(RuntimeOptions.Default); From c79e023c132cc683a17f91e93b14d54ae7abb593 Mon Sep 17 00:00:00 2001 From: KyJah Date: Mon, 30 Sep 2024 11:05:36 -0400 Subject: [PATCH 12/12] dotnet format --- tests/DeltaLake.Tests/Table/LoadTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/DeltaLake.Tests/Table/LoadTests.cs b/tests/DeltaLake.Tests/Table/LoadTests.cs index fa5718b..84df784 100644 --- a/tests/DeltaLake.Tests/Table/LoadTests.cs +++ b/tests/DeltaLake.Tests/Table/LoadTests.cs @@ -1,4 +1,3 @@ -using System.Runtime.InteropServices; using DeltaLake.Errors; using DeltaLake.Runtime; using DeltaLake.Table;