diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 440776f79..7e983172b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -30,6 +30,11 @@ jobs: id: detect-trigger with: keyword: "[test-df-upstream]" + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} # Check crate compiles linux-build-lib: @@ -48,6 +53,11 @@ jobs: run: | cd dask_planner bash update-dependencies.sh + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Check workspace in debug mode run: | cd dask_planner @@ -76,6 +86,11 @@ jobs: run: | cd dask_planner bash update-dependencies.sh + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run tests run: | cd dask_planner diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 6696faad2..64a75f4a7 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -15,6 +15,11 @@ jobs: steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} - uses: actions-rs/toolchain@v1 with: toolchain: nightly diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7189be532..be2d98126 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -109,6 +109,11 @@ jobs: use-mamba: true python-version: "3.8" channel-priority: strict + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install dependencies and nothing else run: | mamba install "setuptools-rust>=1.5.2" diff --git a/continuous_integration/environment-3.10-dev.yaml b/continuous_integration/environment-3.10-dev.yaml index ab6a543ef..078215422 100644 --- a/continuous_integration/environment-3.10-dev.yaml +++ b/continuous_integration/environment-3.10-dev.yaml @@ -32,3 +32,4 @@ dependencies: - tpot - tzlocal>=2.1 - uvicorn>=0.13.4 +- libprotobuf=3 diff --git a/continuous_integration/environment-3.8-dev.yaml b/continuous_integration/environment-3.8-dev.yaml index 94d953dfd..bd2a53afe 100644 --- a/continuous_integration/environment-3.8-dev.yaml +++ b/continuous_integration/environment-3.8-dev.yaml @@ -31,3 +31,4 @@ dependencies: - tpot - tzlocal=2.1 - uvicorn=0.13.4 +- libprotobuf=3 diff --git a/continuous_integration/environment-3.9-dev.yaml b/continuous_integration/environment-3.9-dev.yaml index 04c1d25d7..50270b9b7 100644 --- a/continuous_integration/environment-3.9-dev.yaml +++ b/continuous_integration/environment-3.9-dev.yaml @@ -32,3 +32,4 @@ dependencies: - tpot - tzlocal>=2.1 - uvicorn>=0.13.4 +- libprotobuf=3 diff --git a/continuous_integration/recipe/meta.yaml b/continuous_integration/recipe/meta.yaml index 63c2501fa..94e7bdd50 100644 --- a/continuous_integration/recipe/meta.yaml +++ b/continuous_integration/recipe/meta.yaml @@ -28,6 +28,7 @@ requirements: - pip - python - setuptools-rust >=1.5.2 + - libprotobuf =3 run: - python - dask >=2022.3.0 diff --git a/dask_planner/.cargo/config.toml b/dask_planner/.cargo/config.toml new file mode 100644 index 000000000..d47f983e4 --- /dev/null +++ b/dask_planner/.cargo/config.toml @@ -0,0 +1,11 @@ +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +[target.aarch64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] diff --git a/dask_planner/Cargo.lock b/dask_planner/Cargo.lock index 96d113154..234cc1d08 100644 --- a/dask_planner/Cargo.lock +++ b/dask_planner/Cargo.lock @@ -8,6 +8,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "ahash" version = "0.8.3" @@ -23,9 +29,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] @@ -54,6 +60,39 @@ dependencies = [ "libc", ] +[[package]] +name = "anyhow" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" + +[[package]] +name = "apache-avro" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf4144857f9e4d7dd6cc4ba4c78efd2a46bad682b029bd0d91e76a021af1b2a" +dependencies = [ + "byteorder", + "crc32fast", + "digest", + "lazy_static", + "libflate", + "log", + "num-bigint", + "quad-rand", + "rand", + "regex", + "serde", + "serde_json", + "snap", + "strum", + "strum_macros", + "thiserror", + "typed-builder", + "uuid", + "zerocopy", +] + [[package]] name = "arc-swap" version = "1.6.0" @@ -74,9 +113,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f410d3907b6b3647b9e7bca4551274b2e3d716aa940afb67b7287257401da921" +checksum = "990dfa1a9328504aa135820da1c95066537b69ad94c04881b785f64328e0fa6b" dependencies = [ "ahash", "arrow-arith", @@ -92,14 +131,14 @@ dependencies = [ "arrow-schema", "arrow-select", "arrow-string", - "comfy-table", + "pyo3", ] [[package]] name = "arrow-arith" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87391cf46473c9bc53dab68cb8872c3a81d4dfd1703f1c8aa397dba9880a043" +checksum = "f2b2e52de0ab54173f9b08232b7184c26af82ee7ab4ac77c83396633c90199fa" dependencies = [ "arrow-array", "arrow-buffer", @@ -112,9 +151,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d35d5475e65c57cffba06d0022e3006b677515f99b54af33a7cd54f6cdd4a5b5" +checksum = "e10849b60c17dbabb334be1f4ef7550701aa58082b71335ce1ed586601b2f423" dependencies = [ "ahash", "arrow-buffer", @@ -129,9 +168,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b4ec72eda7c0207727df96cf200f539749d736b21f3e782ece113e18c1a0a7" +checksum = "b0746ae991b186be39933147117f8339eb1c4bbbea1c8ad37e7bf5851a1a06ba" dependencies = [ "half", "num", @@ -139,9 +178,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a7285272c9897321dfdba59de29f5b05aeafd3cdedf104a941256d155f6d304" +checksum = "b88897802515d7b193e38b27ddd9d9e43923d410a9e46307582d756959ee9595" dependencies = [ "arrow-array", "arrow-buffer", @@ -149,15 +188,16 @@ dependencies = [ "arrow-schema", "arrow-select", "chrono", + "comfy-table", "lexical-core", "num", ] [[package]] name = "arrow-csv" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981ee4e7f6a120da04e00d0b39182e1eeacccb59c8da74511de753c56b7fddf7" +checksum = "1c8220d9741fc37961262710ceebd8451a5b393de57c464f0267ffdda1775c0a" dependencies = [ "arrow-array", "arrow-buffer", @@ -174,9 +214,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27cc673ee6989ea6e4b4e8c7d461f7e06026a096c8f0b1a7288885ff71ae1e56" +checksum = "533f937efa1aaad9dc86f6a0e382c2fa736a4943e2090c946138079bdf060cef" dependencies = [ "arrow-buffer", "arrow-schema", @@ -186,9 +226,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e37b8b69d9e59116b6b538e8514e0ec63a30f08b617ce800d31cb44e3ef64c1a" +checksum = "18b75296ff01833f602552dff26a423fc213db8e5049b540ca4a00b1c957e41c" dependencies = [ "arrow-array", "arrow-buffer", @@ -200,9 +240,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c3fa0bed7cfebf6d18e46b733f9cb8a1cb43ce8e6539055ca3e1e48a426266" +checksum = "e501d3de4d612c90677594896ca6c0fa075665a7ff980dc4189bb531c17e19f6" dependencies = [ "arrow-array", "arrow-buffer", @@ -219,23 +259,24 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d247dce7bed6a8d6a3c6debfa707a3a2f694383f0c692a39d736a593eae5ef94" +checksum = "33d2671eb3793f9410230ac3efb0e6d36307be8a2dac5fad58ac9abde8e9f01e" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "half", "num", ] [[package]] name = "arrow-row" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d609c0181f963cea5c70fddf9a388595b5be441f3aa1d1cdbf728ca834bbd3a" +checksum = "fc11fa039338cebbf4e29cf709c8ac1d6a65c7540063d4a25f991ab255ca85c8" dependencies = [ "ahash", "arrow-array", @@ -248,15 +289,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64951898473bfb8e22293e83a44f02874d2257514d49cd95f9aa4afcff183fbc" +checksum = "d04f17f7b86ded0b5baf98fe6123391c4343e031acc3ccc5fa604cc180bff220" +dependencies = [ + "bitflags 2.2.1", +] [[package]] name = "arrow-select" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a513d89c2e1ac22b28380900036cf1f3992c6443efc5e079de631dcf83c6888" +checksum = "163e35de698098ff5f5f672ada9dc1f82533f10407c7a11e2cd09f3bcf31d18a" dependencies = [ "arrow-array", "arrow-buffer", @@ -267,9 +311,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5288979b2705dae1114c864d73150629add9153b9b8f1d7ee3963db94c372ba5" +checksum = "bfdfbed1b10209f0dc68e6aa4c43dc76079af65880965c7c3b73f641f23d4aba" dependencies = [ "arrow-array", "arrow-buffer", @@ -277,7 +321,7 @@ dependencies = [ "arrow-schema", "arrow-select", "regex", - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] @@ -298,6 +342,17 @@ dependencies = [ "zstd-safe 5.0.2+zstd.1.5.2", ] +[[package]] +name = "async-recursion" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + [[package]] name = "async-trait" version = "0.1.68" @@ -306,7 +361,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -327,6 +382,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a6904aef64d73cf10ab17ebace7befb918b82164785cb89907993be7f83813" + [[package]] name = "blake2" version = "0.10.6" @@ -380,11 +441,32 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" +dependencies = [ + "memchr", + "once_cell", + "regex-automata", + "serde", +] + +[[package]] +name = "btoi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd6407f73a9b8b6162d8a2ef999fe6afd7cc15902ebf42c5cd296addf17e0ad" +dependencies = [ + "num-traits", +] + [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" [[package]] name = "byteorder" @@ -441,8 +523,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", + "js-sys", "num-integer", "num-traits", + "serde", + "time 0.1.45", + "wasm-bindgen", "winapi", ] @@ -468,6 +554,12 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "clru" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8191fa7302e03607ff0e237d4246cc043ff5b3cb9409d995172ba3bea16b807" + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -525,9 +617,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" dependencies = [ "libc", ] @@ -602,7 +694,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -619,7 +711,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -640,29 +732,22 @@ name = "dask_planner" version = "0.1.0" dependencies = [ "async-trait", - "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-optimizer", - "datafusion-sql", + "datafusion-python", "env_logger", "log", - "mimalloc", - "parking_lot", "pyo3", + "pyo3-build-config", "pyo3-log", - "rand", - "tokio", - "uuid", ] [[package]] name = "datafusion" -version = "21.1.0" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e29d0f5f8dd3e1c9cdb3acfb137976aad52b16bde42b98ecfe1b323fdc724e23" +checksum = "9bdb93fee4f30368f1f71bfd5cd28882ec9fab0183db7924827b76129d33227c" dependencies = [ "ahash", + "apache-avro", "arrow", "async-compression", "async-trait", @@ -685,6 +770,7 @@ dependencies = [ "itertools", "lazy_static", "log", + "num-traits", "num_cpus", "object_store", "parking_lot", @@ -706,24 +792,26 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "21.1.0" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "191e40fe6afa78bfae747f3f903d330fddf9f4115d059b2a10caee76adee5381" +checksum = "e82401ce129e601d406012b6d718f8978ba84c386e1c342fa155877120d68824" dependencies = [ + "apache-avro", "arrow", "arrow-array", "chrono", "num_cpus", "object_store", "parquet", + "pyo3", "sqlparser", ] [[package]] name = "datafusion-execution" -version = "21.1.0" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3df4bef4a92477d1bf34afe1f0a5eef7c8fdcc2b2c73d69ebc813523bb42b14" +checksum = "b08b2078aed21a27239cd93f3015e492a58b0d50ebeeaf8d2236cf108ef583ce" dependencies = [ "dashmap", "datafusion-common", @@ -739,9 +827,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "21.1.0" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39b02f96264cd469d76e248bf521db04e3d85d52ca1bf2844fb25bee3fa64de2" +checksum = "16b5b977ce9695fb4c67614266ec57f384fc11e9a9f9b3e6d0e62b9c5a9f2c1f" dependencies = [ "ahash", "arrow", @@ -751,9 +839,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "21.1.0" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf8d26b1c85e225cb460f21e9dc9eced6771d8a317244768785c9d8b18686e2" +checksum = "a0b2bb9e73ed778d1bc5af63a270f0154bf6eab5099c77668a6362296888e46b" dependencies = [ "arrow", "async-trait", @@ -764,17 +852,18 @@ dependencies = [ "hashbrown 0.13.2", "itertools", "log", - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] name = "datafusion-physical-expr" -version = "21.1.0" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b6e5d17468514127814addd99414f6dab8ddda6e2fccd922859257a8f618f0" +checksum = "80cd8ea5ab0a07b1b2a3e17d5909f1b1035bd129ffeeb5c66842a32e682f8f79" dependencies = [ "ahash", "arrow", + "arrow-array", "arrow-buffer", "arrow-schema", "blake2", @@ -798,11 +887,37 @@ dependencies = [ "uuid", ] +[[package]] +name = "datafusion-python" +version = "22.0.0" +source = "git+https://github.com/apache/arrow-datafusion-python.git?rev=9493638#94936380e58a266f5dd5de6b70a06d3aa36fbe22" +dependencies = [ + "async-trait", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-optimizer", + "datafusion-sql", + "datafusion-substrait", + "futures", + "mimalloc", + "object_store", + "parking_lot", + "pyo3", + "pyo3-build-config", + "rand", + "regex-syntax 0.6.29", + "syn 2.0.15", + "tokio", + "url", + "uuid", +] + [[package]] name = "datafusion-row" -version = "21.1.0" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3b1f878644f52de5fe52da19a9dd316f8701630803ae3bdb051e52b7a7d9015" +checksum = "2a95d6badab19fd6e9195fdc5209ac0a7e5ce9bcdedc67767b9ffc1b4e645760" dependencies = [ "arrow", "datafusion-common", @@ -812,10 +927,11 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "21.1.0" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f546f7c05ba2398f1e0e422bcd4e7bc1daa54fc8bd75827b0ca1e65af20564e" +checksum = "37a78f8fc67123c4357e63bc0c87622a2a663d26f074958d749a633d0ecde90f" dependencies = [ + "arrow", "arrow-schema", "datafusion-common", "datafusion-expr", @@ -823,6 +939,22 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "datafusion-substrait" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6ed64a2005f0d78f2b1b3ec3f8148183f4523d5d364e5367115f8d8a82b7df" +dependencies = [ + "async-recursion", + "chrono", + "datafusion", + "itertools", + "object_store", + "prost", + "substrait", + "tokio", +] + [[package]] name = "digest" version = "0.10.6" @@ -834,18 +966,59 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "doc-comment" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dunce" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" + +[[package]] +name = "dyn-clone" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" + [[package]] name = "either" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "encoding_rs" +version = "0.8.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +dependencies = [ + "cfg-if", +] + [[package]] name = "env_logger" version = "0.10.0" @@ -889,6 +1062,18 @@ dependencies = [ "instant", ] +[[package]] +name = "filetime" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.2.16", + "windows-sys 0.48.0", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -901,20 +1086,26 @@ version = "23.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77f5399c2c9c50ae9418e522842ad362f61ee48b346ac106807bd355a8a7c619" dependencies = [ - "bitflags", + "bitflags 1.3.2", "rustc_version", ] [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -980,7 +1171,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -1031,146 +1222,817 @@ checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "gix" +version = "0.43.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c256ea71cc1967faaefdaad15f334146b7c806f12460dcafd3afed845c8c78dd" +dependencies = [ + "gix-actor", + "gix-attributes", + "gix-config", + "gix-credentials", + "gix-date", + "gix-diff", + "gix-discover", + "gix-features 0.28.1", + "gix-glob", + "gix-hash 0.10.4", + "gix-hashtable", + "gix-index", + "gix-lock", + "gix-mailmap", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-prompt", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-sec", + "gix-tempfile", + "gix-traverse", + "gix-url", + "gix-validate", + "gix-worktree", + "log", + "once_cell", + "signal-hook", + "smallvec", + "thiserror", + "unicode-normalization", ] [[package]] -name = "glob" -version = "0.3.1" +name = "gix-actor" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "dc22b0cdc52237667c301dd7cdc6ead8f8f73c9f824e9942c8ebd6b764f6c0bf" +dependencies = [ + "bstr", + "btoi", + "gix-date", + "itoa", + "nom", + "thiserror", +] [[package]] -name = "half" -version = "2.2.1" +name = "gix-attributes" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" +checksum = "2231a25934a240d0a4b6f4478401c73ee81d8be52de0293eedbc172334abf3e1" dependencies = [ - "crunchy", - "num-traits", + "bstr", + "gix-features 0.28.1", + "gix-glob", + "gix-path", + "gix-quote", + "thiserror", + "unicode-bom", ] [[package]] -name = "hashbrown" -version = "0.12.3" +name = "gix-bitmap" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +checksum = "55a95f4942360766c3880bdb2b4b57f1ef73b190fc424755e7fdf480430af618" +dependencies = [ + "thiserror", +] [[package]] -name = "hashbrown" -version = "0.13.2" +name = "gix-chunk" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +checksum = "b0d39583cab06464b8bf73b3f1707458270f0e7383cb24c3c9c1a16e6f792978" dependencies = [ - "ahash", + "thiserror", ] [[package]] -name = "heck" -version = "0.4.1" +name = "gix-command" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "b2c6f75c1e0f924de39e750880a6e21307194bb1ab773efe3c7d2d787277f8ab" +dependencies = [ + "bstr", +] [[package]] -name = "hermit-abi" -version = "0.2.6" +name = "gix-config" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +checksum = "7fbad5ce54a8fc997acc50febd89ec80fa6e97cb7f8d0654cb229936407489d8" dependencies = [ - "libc", + "bstr", + "gix-config-value", + "gix-features 0.28.1", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "log", + "memchr", + "nom", + "once_cell", + "smallvec", + "thiserror", + "unicode-bom", ] [[package]] -name = "hermit-abi" -version = "0.3.1" +name = "gix-config-value" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "d09154c0c8677e4da0ec35e896f56ee3e338e741b9599fae06075edd83a4081c" +dependencies = [ + "bitflags 1.3.2", + "bstr", + "gix-path", + "libc", + "thiserror", +] [[package]] -name = "humantime" -version = "2.1.0" +name = "gix-credentials" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "750b684197374518ea057e0a0594713e07683faa0a3f43c0f93d97f64130ad8d" +dependencies = [ + "bstr", + "gix-command", + "gix-config-value", + "gix-path", + "gix-prompt", + "gix-sec", + "gix-url", + "thiserror", +] [[package]] -name = "iana-time-zone" -version = "0.1.56" +name = "gix-date" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +checksum = "b96271912ce39822501616f177dea7218784e6c63be90d5f36322ff3a722aae2" dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows", + "bstr", + "itoa", + "thiserror", + "time 0.3.20", ] [[package]] -name = "iana-time-zone-haiku" -version = "0.1.1" +name = "gix-diff" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "103a0fa79b0d438f5ecb662502f052e530ace4fe1fe8e1c83c0c6da76d728e67" dependencies = [ - "cxx", - "cxx-build", + "gix-hash 0.10.4", + "gix-object", + "imara-diff", + "thiserror", ] [[package]] -name = "idna" -version = "0.3.0" +name = "gix-discover" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "6eba8ba458cb8f4a6c33409b0fe650b1258655175a7ffd1d24fafd3ed31d880b" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "bstr", + "dunce", + "gix-hash 0.10.4", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror", ] [[package]] -name = "indexmap" -version = "1.9.3" +name = "gix-features" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "0b76f9a80f6dd7be66442ae86e1f534effad9546676a392acc95e269d0c21c22" dependencies = [ - "autocfg", - "hashbrown 0.12.3", + "crc32fast", + "flate2", + "gix-hash 0.10.4", + "libc", + "once_cell", + "prodash", + "sha1_smol", + "thiserror", + "walkdir", ] [[package]] -name = "indoc" -version = "1.0.9" +name = "gix-features" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "cf69b0f5c701cc3ae22d3204b671907668f6437ca88862d355eaf9bc47a4f897" +dependencies = [ + "gix-hash 0.11.1", + "libc", +] [[package]] -name = "instant" -version = "0.1.12" +name = "gix-fs" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "9b37a1832f691fdc09910bd267f9a2e413737c1f9ec68c6e31f9e802616278a9" dependencies = [ - "cfg-if", + "gix-features 0.29.0", ] [[package]] -name = "integer-encoding" -version = "3.0.4" +name = "gix-glob" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +checksum = "93e43efd776bc543f46f0fd0ca3d920c37af71a764a16f2aebd89765e9ff2993" +dependencies = [ + "bitflags 1.3.2", + "bstr", +] [[package]] -name = "io-lifetimes" -version = "1.0.10" +name = "gix-hash" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "2a258595457bc192d1f1c59d0d168a1e34e2be9b97a614e14995416185de41a7" dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.48.0", + "hex", + "thiserror", ] [[package]] -name = "is-terminal" -version = "0.4.7" +name = "gix-hash" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078eec3ac2808cc03f0bddd2704cb661da5c5dc33b41a9d7947b141d499c7c42" +dependencies = [ + "hex", + "thiserror", +] + +[[package]] +name = "gix-hashtable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e55e40dfd694884f0eb78796c5bddcf2f8b295dace47039099dd7e76534973" +dependencies = [ + "gix-hash 0.10.4", + "hashbrown 0.13.2", + "parking_lot", +] + +[[package]] +name = "gix-index" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "717ab601ece7921f59fe86849dbe27d44a46ebb883b5885732c4f30df4996177" +dependencies = [ + "bitflags 1.3.2", + "bstr", + "btoi", + "filetime", + "gix-bitmap", + "gix-features 0.28.1", + "gix-hash 0.10.4", + "gix-lock", + "gix-object", + "gix-traverse", + "itoa", + "memmap2", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-lock" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c693d7f05730fa74a7c467150adc7cea393518410c65f0672f80226b8111555" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror", +] + +[[package]] +name = "gix-mailmap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b66aea5e52875cd4915f4957a6f4b75831a36981e2ec3f5fad9e370e444fe1a" +dependencies = [ + "bstr", + "gix-actor", + "thiserror", +] + +[[package]] +name = "gix-object" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df068db9180ee935fbb70504848369e270bdcb576b05c0faa8b9fd3b86fc017" +dependencies = [ + "bstr", + "btoi", + "gix-actor", + "gix-features 0.28.1", + "gix-hash 0.10.4", + "gix-validate", + "hex", + "itoa", + "nom", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-odb" +version = "0.43.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83af2e3e36005bfe010927f0dff41fb5acc3e3d89c6f1174135b3a34086bda2" +dependencies = [ + "arc-swap", + "gix-features 0.28.1", + "gix-hash 0.10.4", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "parking_lot", + "tempfile", + "thiserror", +] + +[[package]] +name = "gix-pack" +version = "0.33.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9401911c7fe032ad7b31c6a6b5be59cb283d1d6c999417a8215056efe6d635f3" +dependencies = [ + "clru", + "gix-chunk", + "gix-diff", + "gix-features 0.28.1", + "gix-hash 0.10.4", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-traverse", + "memmap2", + "parking_lot", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-path" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32370dce200bb951df013e03dff35b4233fc7a89458642b047629b91734a7e19" +dependencies = [ + "bstr", + "thiserror", +] + +[[package]] +name = "gix-prompt" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3034d4d935aef2c7bf719aaa54b88c520e82413118d886ae880a31d5bdee57" +dependencies = [ + "gix-command", + "gix-config-value", + "nix", + "parking_lot", + "thiserror", +] + +[[package]] +name = "gix-quote" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a282f5a8d9ee0b09ec47390ac727350c48f2f5c76d803cd8da6b3e7ad56e0bcb" +dependencies = [ + "bstr", + "btoi", + "thiserror", +] + +[[package]] +name = "gix-ref" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e909396ed3b176823991ccc391c276ae2a015e54edaafa3566d35123cfac9d" +dependencies = [ + "gix-actor", + "gix-features 0.28.1", + "gix-hash 0.10.4", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-validate", + "memmap2", + "nom", + "thiserror", +] + +[[package]] +name = "gix-refspec" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aba332462bda2e8efeae4302b39a6ed01ad56ef772fd5b7ef197cf2798294d65" +dependencies = [ + "bstr", + "gix-hash 0.10.4", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-revision" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c6f6ff53f888858afc24bf12628446a14279ceec148df6194481f306f553ad2" +dependencies = [ + "bstr", + "gix-date", + "gix-hash 0.10.4", + "gix-hashtable", + "gix-object", + "thiserror", +] + +[[package]] +name = "gix-sec" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8ffa5bf0772f9b01de501c035b6b084cf9b8bb07dec41e3afc6a17336a65f47" +dependencies = [ + "bitflags 1.3.2", + "dirs", + "gix-path", + "libc", + "windows 0.43.0", +] + +[[package]] +name = "gix-tempfile" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71a0d32f34e71e86586124225caefd78dabc605d0486de580d717653addf182" +dependencies = [ + "gix-fs", + "libc", + "once_cell", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-traverse" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd9a4a07bb22168dc79c60e1a6a41919d198187ca83d8a5940ad8d7122a45df3" +dependencies = [ + "gix-hash 0.10.4", + "gix-hashtable", + "gix-object", + "thiserror", +] + +[[package]] +name = "gix-url" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6a22b4b32ad14d68f7b7fb6458fa58d44b01797d94c1b8f4db2d9c7b3c366b5" +dependencies = [ + "bstr", + "gix-features 0.28.1", + "gix-path", + "home", + "thiserror", + "url", +] + +[[package]] +name = "gix-utils" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c10b69beac219acb8df673187a1f07dde2d74092f974fb3f9eb385aeb667c909" +dependencies = [ + "fastrand", +] + +[[package]] +name = "gix-validate" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd629d3680773e1785e585d76fd4295b740b559cad9141517300d99a0c8c049" +dependencies = [ + "bstr", + "thiserror", +] + +[[package]] +name = "gix-worktree" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ec9a000b4f24af706c3cc680c7cda235656cbe3216336522f5692773b8a301" +dependencies = [ + "bstr", + "gix-attributes", + "gix-features 0.28.1", + "gix-glob", + "gix-hash 0.10.4", + "gix-index", + "gix-object", + "gix-path", + "io-close", + "thiserror", +] + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "h2" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" +dependencies = [ + "crunchy", + "num-traits", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +dependencies = [ + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows 0.48.0", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "imara-diff" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e98c1d0ad70fc91b8b9654b1f33db55e59579d3b3de2bffdced0fdb810570cb8" +dependencies = [ + "ahash", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indoc" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "ipnet" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" + +[[package]] +name = "is-terminal" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ @@ -1285,9 +2147,29 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.141" +version = "0.2.142" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" + +[[package]] +name = "libflate" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97822bf791bd4d5b403713886a5fbe8bf49520fe78e323b0dc480ca1a03e50b0" +dependencies = [ + "adler32", + "crc32fast", + "libflate_lz77", +] + +[[package]] +name = "libflate_lz77" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf" +dependencies = [ + "rle-decode-fast", +] [[package]] name = "libm" @@ -1297,9 +2179,9 @@ checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" [[package]] name = "libmimalloc-sys" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a558e3d911bc3c7bfc8c78bc580b404d6e51c1cefbf656e176a94b49b0df40" +checksum = "f4ac0e912c8ef1b735e92369695618dc5b1819f5a7bf3f167301a3ba1cea515e" dependencies = [ "cc", "libc", @@ -1316,9 +2198,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "b64f40e5e03e0d54f03845c8197d0291253cdbedfb1cb46b13c2c117554a9f4c" [[package]] name = "lock_api" @@ -1385,6 +2267,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.8.0" @@ -1396,22 +2287,74 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.36" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d88dad3f985ec267a3fcb7a1726f5cb1a7e8cad8b646e70a84f967210df23da" +checksum = "4e2894987a3459f3ffb755608bd82188f8ed00d0ae077f1edea29c068d639d98" dependencies = [ "libmimalloc-sys", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.45.0", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "nix" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "static_assertions", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num" version = "0.4.0" @@ -1499,6 +2442,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "object_store" version = "0.5.6" @@ -1506,12 +2458,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec9cd6ca25e796a49fa242876d1c4de36a24a6da5258e9f0bc062dbf5e81c53b" dependencies = [ "async-trait", + "base64", "bytes", "chrono", "futures", "itertools", "parking_lot", "percent-encoding", + "quick-xml", + "rand", + "reqwest", + "ring", + "rustls-pemfile", + "serde", + "serde_json", "snafu", "tokio", "tracing", @@ -1559,9 +2519,9 @@ dependencies = [ [[package]] name = "parquet" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ac135ecf63ebb5f53dda0921b0b76d6048b3ef631a5f4760b9e8f863ff00cfa" +checksum = "321a15f8332645759f29875b07f8233d16ed8ec1b3582223de81625a9f8506b7" dependencies = [ "ahash", "arrow-array", @@ -1683,6 +2643,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "prettyplease" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ceca8aaf45b5c46ec7ed39fff75f57290368c1846d33d24a122ca81416ab058" +dependencies = [ + "proc-macro2", + "syn 2.0.15", +] + [[package]] name = "proc-macro-hack" version = "0.5.20+deprecated" @@ -1698,11 +2668,69 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prodash" +version = "23.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9516b775656bc3e8985e19cd4b8c0c0de045095074e453d2c0a513b5f978392d" + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "regex", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + [[package]] name = "pyo3" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb848f80438f926a9ebddf0a539ed6065434fd7aae03a89312a9821f81b8501" +checksum = "e3b1ac5b3731ba34fdaa9785f8d74d17448cd18f30cf19e0c7e7b1fdb5272109" dependencies = [ "cfg-if", "indoc", @@ -1717,9 +2745,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98a42e7f42e917ce6664c832d5eee481ad514c98250c49e0b03b20593e2c7ed0" +checksum = "9cb946f5ac61bb61a5014924910d936ebd2b23b705f7a4a3c40b05c720b079a3" dependencies = [ "once_cell", "target-lexicon", @@ -1727,9 +2755,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0707f0ab26826fe4ccd59b69106e9df5e12d097457c7b8f9c0fd1d2743eec4d" +checksum = "fd4d7c5337821916ea2a1d21d1092e8443cf34879e53a0ac653fbb98f44ff65c" dependencies = [ "libc", "pyo3-build-config", @@ -1748,9 +2776,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978d18e61465ecd389e1f235ff5a467146dc4e3c3968b90d274fe73a5dd4a438" +checksum = "a9d39c55dab3fc5a4b25bbd1ac10a2da452c4aca13bb450f22818a002e29648d" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1760,15 +2788,31 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e0e1128f85ce3fca66e435e08aa2089a2689c1c48ce97803e13f63124058462" +checksum = "97daff08a4c48320587b5224cc98d609e3c27b6d437315bd40b605c98eeb5918" dependencies = [ "proc-macro2", "quote", "syn 1.0.109", ] +[[package]] +name = "quad-rand" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" + +[[package]] +name = "quick-xml" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.26" @@ -1814,7 +2858,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1823,26 +2867,121 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall 0.2.16", + "thiserror", ] [[package]] name = "regex" -version = "1.7.3" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.1", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + [[package]] name = "regex-syntax" version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" + +[[package]] +name = "regress" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d995d590bd8ec096d1893f414bf3f5e8b0ee4c9eed9a5642b9766ef2c8e2e8e9" +dependencies = [ + "hashbrown 0.13.2", + "memchr", +] + +[[package]] +name = "reqwest" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13293b639a097af28fc8a90f22add145a9c954e49d77da06263d58cf44d5fb91" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-rustls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + [[package]] name = "rustc_version" version = "0.4.0" @@ -1854,11 +2993,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.11" +version = "0.37.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +checksum = "8bbfc1d1c7c40c01715f47d71444744a81669ca84e8b63e25a55e169b1f86433" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -1866,6 +3005,27 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rustls" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +dependencies = [ + "base64", +] + [[package]] name = "rustversion" version = "1.0.12" @@ -1887,6 +3047,30 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schemars" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c613288622e5f0c3fdc5dbd4db1c5fbe752746b1d1a56a0630b78fd00de44f" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -1900,34 +3084,111 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] -name = "semver" -version = "1.0.17" +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "semver" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" + +[[package]] +name = "seq-macro" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc" + +[[package]] +name = "serde" +version = "1.0.160" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.160" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "serde_json" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +dependencies = [ + "itoa", + "ryu", + "serde", +] [[package]] -name = "seq-macro" -version = "0.3.3" +name = "serde_tokenstream" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc" +checksum = "797ba1d80299b264f3aac68ab5d12e5825a561749db4df7cd7c8083900c5d4e9" +dependencies = [ + "proc-macro2", + "serde", + "syn 1.0.109", +] [[package]] -name = "serde" -version = "1.0.159" +name = "serde_urlencoded" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] [[package]] -name = "serde_json" -version = "1.0.95" +name = "serde_yaml" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" +checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" dependencies = [ + "indexmap", "itoa", "ryu", "serde", + "unsafe-libyaml", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.10.6" @@ -1939,6 +3200,25 @@ dependencies = [ "digest", ] +[[package]] +name = "signal-hook" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + [[package]] name = "siphasher" version = "0.3.10" @@ -1988,6 +3268,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "sqlparser" version = "0.32.0" @@ -2034,6 +3330,28 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "substrait" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ae64fb7ad0670c7d6d53d57b1b91beb2212afc30e164cc8edb02d6b2cff32a" +dependencies = [ + "gix", + "heck", + "prettyplease", + "prost", + "prost-build", + "prost-types", + "schemars", + "semver", + "serde", + "serde_json", + "serde_yaml", + "syn 2.0.15", + "typify", + "walkdir", +] + [[package]] name = "subtle" version = "2.4.1" @@ -2053,9 +3371,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.13" +version = "2.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" dependencies = [ "proc-macro2", "quote", @@ -2064,9 +3382,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.6" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" +checksum = "fd1ba337640d60c3e96bc6f0638a939b9c9a7f2c316a1598c279828b3d1dc8c5" [[package]] name = "tempfile" @@ -2090,6 +3408,26 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + [[package]] name = "thrift" version = "0.17.0" @@ -2101,6 +3439,46 @@ dependencies = [ "ordered-float", ] +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "time" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +dependencies = [ + "itoa", + "libc", + "num_threads", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +dependencies = [ + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -2127,35 +3505,49 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.27.0" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" dependencies = [ "autocfg", "bytes", + "libc", + "mio", "num_cpus", "parking_lot", "pin-project-lite", + "socket2", "tokio-macros", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", +] + +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls", + "tokio", + "webpki", ] [[package]] name = "tokio-stream" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", @@ -2164,17 +3556,24 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", + "tracing", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -2189,13 +3588,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] @@ -2207,6 +3606,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "twox-hash" version = "1.6.3" @@ -2217,18 +3622,79 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "typed-builder" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "typenum" version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +[[package]] +name = "typify" +version = "0.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30bfde96849e25d7feef1bbf652e9cfc51deb63203fdc07b115b8bc3bcfe20b9" +dependencies = [ + "typify-impl", + "typify-macro", +] + +[[package]] +name = "typify-impl" +version = "0.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95d27d749378ceab6ec22188ed7ad102205c89ddb92ab662371c850ffc71aa1a" +dependencies = [ + "heck", + "log", + "proc-macro2", + "quote", + "regress", + "schemars", + "serde_json", + "syn 1.0.109", + "thiserror", + "unicode-ident", +] + +[[package]] +name = "typify-macro" +version = "0.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35db6fc2bd9220ecdac6eeb88158824b83610de3dda0c6d0f2142b49efd858b0" +dependencies = [ + "proc-macro2", + "quote", + "schemars", + "serde", + "serde_json", + "serde_tokenstream", + "syn 1.0.109", + "typify-impl", +] + [[package]] name = "unicode-bidi" version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +[[package]] +name = "unicode-bom" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63ec69f541d875b783ca40184d655f2927c95f0bffd486faa83cd3ac3529ec32" + [[package]] name = "unicode-ident" version = "1.0.8" @@ -2262,6 +3728,18 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +[[package]] +name = "unsafe-libyaml" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.3.1" @@ -2275,11 +3753,12 @@ dependencies = [ [[package]] name = "uuid" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" +checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" dependencies = [ "getrandom", + "serde", ] [[package]] @@ -2298,6 +3777,22 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2329,6 +3824,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.84" @@ -2358,6 +3865,59 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +[[package]] +name = "wasm-streams" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + +[[package]] +name = "which" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +dependencies = [ + "either", + "libc", + "once_cell", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2389,6 +3949,21 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04662ed0e3e5630dfa9b26e4cb823b817f1a9addda855d973a9458c236556244" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows" version = "0.48.0" @@ -2530,6 +4105,15 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "xz2" version = "0.1.7" @@ -2539,6 +4123,27 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "zerocopy" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332f188cc1bcf1fe1064b8c58d150f497e697f49774aa846f2dc949d9a25f236" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6505e6815af7de1746a08f69c69606bb45695a17149517680f3b2149713b19a3" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" diff --git a/dask_planner/Cargo.toml b/dask_planner/Cargo.toml index af63486a1..0e45b0732 100644 --- a/dask_planner/Cargo.toml +++ b/dask_planner/Cargo.toml @@ -10,20 +10,14 @@ rust-version = "1.65" [dependencies] async-trait = "0.1.68" -datafusion = "21.1.0" -datafusion-common = "21.1.0" -datafusion-expr = "21.1.0" -datafusion-optimizer = "21.1.0" -datafusion-sql = "21.1.0" +datafusion-python = { git = "https://github.com/apache/arrow-datafusion-python.git", rev = "9493638" } env_logger = "0.10" log = "^0.4" -mimalloc = { version = "*", default-features = false } -parking_lot = "0.12" -pyo3 = { version = "0.18.2", features = ["extension-module", "abi3", "abi3-py38"] } +pyo3 = { version = "0.18.3", features = ["extension-module", "abi3", "abi3-py38"] } pyo3-log = "0.8.1" -rand = "0.8" -tokio = { version = "1.27", features = ["macros", "rt", "rt-multi-thread", "sync", "fs", "parking_lot"] } -uuid = { version = "1.3", features = ["v4"] } + +[build-dependencies] +pyo3-build-config = "0.18.3" [lib] crate-type = ["cdylib"] diff --git a/dask_planner/src/dialect.rs b/dask_planner/src/dialect.rs index 95bb8858f..9fe013f3d 100644 --- a/dask_planner/src/dialect.rs +++ b/dask_planner/src/dialect.rs @@ -1,6 +1,6 @@ use core::{iter::Peekable, str::Chars}; -use datafusion_sql::sqlparser::{ +use datafusion_python::datafusion_sql::sqlparser::{ ast::{Expr, Function, FunctionArg, FunctionArgExpr, Ident, ObjectName, Value}, dialect::Dialect, keywords::Keyword, diff --git a/dask_planner/src/error.rs b/dask_planner/src/error.rs index d5b0eb39c..e21633cc5 100644 --- a/dask_planner/src/error.rs +++ b/dask_planner/src/error.rs @@ -1,7 +1,9 @@ use std::fmt::{Display, Formatter}; -use datafusion_common::DataFusionError; -use datafusion_sql::sqlparser::{parser::ParserError, tokenizer::TokenizerError}; +use datafusion_python::{ + datafusion_common::DataFusionError, + datafusion_sql::sqlparser::{parser::ParserError, tokenizer::TokenizerError}, +}; use pyo3::PyErr; pub type Result = std::result::Result; diff --git a/dask_planner/src/expression.rs b/dask_planner/src/expression.rs index 291586ddc..150d332b8 100644 --- a/dask_planner/src/expression.rs +++ b/dask_planner/src/expression.rs @@ -1,21 +1,23 @@ use std::{borrow::Cow, convert::From, sync::Arc}; -use datafusion::arrow::datatypes::DataType; -use datafusion_common::{Column, DFField, DFSchema, ScalarValue}; -use datafusion_expr::{ - expr::{AggregateFunction, BinaryExpr, Cast, Sort, TryCast, WindowFunction}, - lit, - utils::exprlist_to_fields, - Between, - BuiltinScalarFunction, - Case, - Expr, - GetIndexedField, - Like, - LogicalPlan, - Operator, +use datafusion_python::{ + datafusion::arrow::datatypes::DataType, + datafusion_common::{Column, DFField, DFSchema, ScalarValue}, + datafusion_expr::{ + expr::{AggregateFunction, BinaryExpr, Cast, Sort, TryCast, WindowFunction}, + lit, + utils::exprlist_to_fields, + Between, + BuiltinScalarFunction, + Case, + Expr, + GetIndexedField, + Like, + LogicalPlan, + Operator, + }, + datafusion_sql::TableReference, }; -use datafusion_sql::TableReference; use pyo3::prelude::*; use crate::{ @@ -759,6 +761,21 @@ impl PyExpr { } } + #[pyo3(name = "getIntervalMonthDayNanoValue")] + pub fn interval_month_day_nano_value(&self) -> PyResult> { + match self.get_scalar_value()? { + ScalarValue::IntervalMonthDayNano(Some(iv)) => { + let interval = *iv as u128; + let months = (interval >> 32) as i32; + let days = (interval >> 64) as i32; + let ns = interval as i64; + Ok(Some((months, days, ns))) + } + ScalarValue::IntervalMonthDayNano(None) => Ok(None), + other => Err(unexpected_literal_value(other)), + } + } + #[pyo3(name = "isNegated")] pub fn is_negated(&self) -> PyResult { match &self.expr { @@ -873,8 +890,10 @@ pub fn expr_to_field(expr: &Expr, input_plan: &LogicalPlan) -> Result { #[cfg(test)] mod test { - use datafusion_common::{Column, ScalarValue}; - use datafusion_expr::Expr; + use datafusion_python::{ + datafusion_common::{Column, ScalarValue}, + datafusion_expr::Expr, + }; use crate::{error::Result, expression::PyExpr}; diff --git a/dask_planner/src/lib.rs b/dask_planner/src/lib.rs index c390fe5f9..f5305d900 100644 --- a/dask_planner/src/lib.rs +++ b/dask_planner/src/lib.rs @@ -1,5 +1,4 @@ use log::debug; -use mimalloc::MiMalloc; use pyo3::prelude::*; mod dialect; @@ -8,9 +7,6 @@ mod expression; mod parser; mod sql; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - /// Low-level DataFusion internal package. /// /// The higher-level public API is defined in pure python files under the diff --git a/dask_planner/src/parser.rs b/dask_planner/src/parser.rs index c07f3e1e8..3147e6309 100644 --- a/dask_planner/src/parser.rs +++ b/dask_planner/src/parser.rs @@ -4,7 +4,7 @@ use std::collections::VecDeque; -use datafusion_sql::sqlparser::{ +use datafusion_python::datafusion_sql::sqlparser::{ ast::{Expr, Ident, SelectItem, Statement as SQLStatement, UnaryOperator, Value}, dialect::{keywords::Keyword, Dialect}, parser::{Parser, ParserError}, diff --git a/dask_planner/src/sql.rs b/dask_planner/src/sql.rs index 648fe62ce..22f6d01ac 100644 --- a/dask_planner/src/sql.rs +++ b/dask_planner/src/sql.rs @@ -11,28 +11,34 @@ pub mod types; use std::{collections::HashMap, sync::Arc}; -use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; -use datafusion_common::{config::ConfigOptions, DFSchema, DataFusionError}; -use datafusion_expr::{ - logical_plan::Extension, - AccumulatorFunctionImplementation, - AggregateUDF, - LogicalPlan, - PlanVisitor, - ReturnTypeFunction, - ScalarFunctionImplementation, - ScalarUDF, - Signature, - StateTypeFunction, - TableSource, - TypeSignature, - Volatility, -}; -use datafusion_sql::{ - parser::Statement as DFStatement, - planner::{ContextProvider, SqlToRel}, - ResolvedTableReference, - TableReference, +use datafusion_python::{ + datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}, + datafusion_common::{ + config::ConfigOptions, + tree_node::{TreeNode, TreeNodeVisitor, VisitRecursion}, + DFSchema, + DataFusionError, + }, + datafusion_expr::{ + logical_plan::Extension, + AccumulatorFunctionImplementation, + AggregateUDF, + LogicalPlan, + ReturnTypeFunction, + ScalarFunctionImplementation, + ScalarUDF, + Signature, + StateTypeFunction, + TableSource, + TypeSignature, + Volatility, + }, + datafusion_sql::{ + parser::Statement as DFStatement, + planner::{ContextProvider, SqlToRel}, + ResolvedTableReference, + TableReference, + }, }; use log::{debug, warn}; use pyo3::prelude::*; @@ -74,9 +80,9 @@ use crate::{ /// from SQL using DaskSQLContext. /// /// ``` -/// use datafusion::prelude::*; +/// use datafusion_python::datafusion::prelude::*; /// -/// # use datafusion_common::Result; +/// # use datafusion_python::datafusion_common::Result; /// # #[tokio::main] /// # async fn main() -> Result<()> { /// let mut ctx = DaskSQLContext::new(); @@ -190,7 +196,10 @@ impl ContextProvider for DaskSQLContext { match name { "year" => { - let sig = generate_signatures(vec![numeric_datatypes]); + let sig = Signature::exact( + vec![DataType::Timestamp(TimeUnit::Nanosecond, None)], + Volatility::Immutable, + ); let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Int64))); return Some(Arc::new(ScalarUDF::new(name, &sig, &rtf, &fun))); } @@ -204,8 +213,18 @@ impl ContextProvider for DaskSQLContext { return Some(Arc::new(ScalarUDF::new(name, &sig, &rtf, &fun))); } "timestampceil" | "timestampfloor" => { - let sig = Signature::exact( - vec![DataType::Date64, DataType::Utf8], + // let sig = Signature::exact( + // vec![DataType::Timestamp(TimeUnit::Nanosecond, None), DataType::Date64, DataType::Utf8], + // Volatility::Immutable, + // ); + let sig = Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Date64, DataType::Utf8]), + TypeSignature::Exact(vec![ + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Utf8, + ]), + ], Volatility::Immutable, ); let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Date64))); @@ -356,7 +375,8 @@ impl ContextProvider for DaskSQLContext { match name { "every" => { - let sig = generate_signatures(vec![numeric_datatypes]); + // let sig = generate_signatures(vec![DataType::Boolean]); + let sig = Signature::exact(vec![DataType::Boolean], Volatility::Immutable); let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Boolean))); return Some(Arc::new(AggregateUDF::new(name, &sig, &rtf, &acc, &st))); } @@ -518,20 +538,21 @@ impl DaskSQLContext { // Certain queries cannot be optimized. Ex: `EXPLAIN SELECT * FROM test` simply return those plans as is let mut visitor = OptimizablePlanVisitor {}; - match existing_plan.original_plan.accept(&mut visitor) { + match existing_plan.original_plan.visit(&mut visitor) { Ok(valid) => { - if valid { - optimizer::DaskSqlOptimizer::new() + match valid { + VisitRecursion::Stop => { + // This LogicalPlan does not support Optimization. Return original + warn!("This LogicalPlan does not support Optimization. Returning original"); + Ok(existing_plan) + } + _ => optimizer::DaskSqlOptimizer::new() .optimize(existing_plan.original_plan) .map(|k| PyLogicalPlan { original_plan: k, current_node: None, }) - .map_err(py_optimization_exp) - } else { - // This LogicalPlan does not support Optimization. Return original - warn!("This LogicalPlan does not support Optimization. Returning original"); - Ok(existing_plan) + .map_err(py_optimization_exp), } } Err(e) => Err(py_optimization_exp(e)), @@ -694,19 +715,19 @@ impl DaskSQLContext { /// Visits each AST node to determine if the plan is valid for optimization or not pub struct OptimizablePlanVisitor; -impl PlanVisitor for OptimizablePlanVisitor { - type Error = DataFusionError; +impl TreeNodeVisitor for OptimizablePlanVisitor { + type N = LogicalPlan; - fn pre_visit(&mut self, plan: &LogicalPlan) -> std::result::Result { + fn pre_visit(&mut self, plan: &LogicalPlan) -> Result { // If the plan contains an unsupported Node type we flag the plan as un-optimizable here match plan { - LogicalPlan::Explain(..) => Ok(false), - _ => Ok(true), + LogicalPlan::Explain(..) => Ok(VisitRecursion::Stop), + _ => Ok(VisitRecursion::Continue), } } - fn post_visit(&mut self, _plan: &LogicalPlan) -> std::result::Result { - Ok(true) + fn post_visit(&mut self, _plan: &LogicalPlan) -> Result { + Ok(VisitRecursion::Continue) } } @@ -743,8 +764,10 @@ fn generate_signatures(cartesian_setup: Vec>) -> Signature { #[cfg(test)] mod test { - use datafusion::arrow::datatypes::DataType; - use datafusion_expr::{Signature, TypeSignature, Volatility}; + use datafusion_python::{ + datafusion::arrow::datatypes::DataType, + datafusion_expr::{Signature, TypeSignature, Volatility}, + }; use crate::sql::generate_signatures; diff --git a/dask_planner/src/sql/column.rs b/dask_planner/src/sql/column.rs index 1c6839755..63f043901 100644 --- a/dask_planner/src/sql/column.rs +++ b/dask_planner/src/sql/column.rs @@ -1,4 +1,4 @@ -use datafusion_common::Column; +use datafusion_python::datafusion_common::Column; use pyo3::prelude::*; #[pyclass(name = "Column", module = "dask_planner", subclass)] diff --git a/dask_planner/src/sql/function.rs b/dask_planner/src/sql/function.rs index b8bd2b5c7..39fa7635e 100644 --- a/dask_planner/src/sql/function.rs +++ b/dask_planner/src/sql/function.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use datafusion::arrow::datatypes::DataType; +use datafusion_python::datafusion::arrow::datatypes::DataType; use pyo3::prelude::*; use super::types::PyDataType; diff --git a/dask_planner/src/sql/logical.rs b/dask_planner/src/sql/logical.rs index b84b51bd6..d2096ba9b 100644 --- a/dask_planner/src/sql/logical.rs +++ b/dask_planner/src/sql/logical.rs @@ -35,8 +35,10 @@ pub mod table_scan; pub mod use_schema; pub mod window; -use datafusion_common::{DFSchemaRef, DataFusionError}; -use datafusion_expr::LogicalPlan; +use datafusion_python::{ + datafusion_common::{DFSchemaRef, DataFusionError}, + datafusion_expr::LogicalPlan, +}; use pyo3::prelude::*; use self::{ @@ -325,7 +327,7 @@ impl PyLogicalPlan { LogicalPlan::CreateCatalogSchema(_create) => "CreateCatalogSchema", LogicalPlan::CreateCatalog(_create_catalog) => "CreateCatalog", LogicalPlan::CreateView(_create_view) => "CreateView", - LogicalPlan::SetVariable(_) => "SetVariable", + LogicalPlan::Statement(_) => "Statement", // Further examine and return the name that is a possible Dask-SQL Extension type LogicalPlan::Extension(extension) => { let node = extension.node.as_any(); diff --git a/dask_planner/src/sql/logical/aggregate.rs b/dask_planner/src/sql/logical/aggregate.rs index 101203c6f..0acc8b86e 100644 --- a/dask_planner/src/sql/logical/aggregate.rs +++ b/dask_planner/src/sql/logical/aggregate.rs @@ -1,4 +1,4 @@ -use datafusion_expr::{ +use datafusion_python::datafusion_expr::{ expr::AggregateFunction, logical_plan::{Aggregate, Distinct}, Expr, diff --git a/dask_planner/src/sql/logical/alter_schema.rs b/dask_planner/src/sql/logical/alter_schema.rs index 3345a66f6..742ae513f 100644 --- a/dask_planner/src/sql/logical/alter_schema.rs +++ b/dask_planner/src/sql/logical/alter_schema.rs @@ -5,11 +5,13 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{ - logical_plan::{Extension, UserDefinedLogicalNode}, - Expr, - LogicalPlan, +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{ + logical_plan::{Extension, UserDefinedLogicalNode}, + Expr, + LogicalPlan, + }, }; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/alter_table.rs b/dask_planner/src/sql/logical/alter_table.rs index fab0bbd8a..7f51a15c3 100644 --- a/dask_planner/src/sql/logical/alter_table.rs +++ b/dask_planner/src/sql/logical/alter_table.rs @@ -5,11 +5,13 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{ - logical_plan::{Extension, UserDefinedLogicalNode}, - Expr, - LogicalPlan, +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{ + logical_plan::{Extension, UserDefinedLogicalNode}, + Expr, + LogicalPlan, + }, }; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/analyze_table.rs b/dask_planner/src/sql/logical/analyze_table.rs index b04d7d78c..9fa7fb219 100644 --- a/dask_planner/src/sql/logical/analyze_table.rs +++ b/dask_planner/src/sql/logical/analyze_table.rs @@ -5,11 +5,13 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{ - logical_plan::{Extension, UserDefinedLogicalNode}, - Expr, - LogicalPlan, +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{ + logical_plan::{Extension, UserDefinedLogicalNode}, + Expr, + LogicalPlan, + }, }; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/create_catalog_schema.rs b/dask_planner/src/sql/logical/create_catalog_schema.rs index 2ef3b65b4..bc89b02ce 100644 --- a/dask_planner/src/sql/logical/create_catalog_schema.rs +++ b/dask_planner/src/sql/logical/create_catalog_schema.rs @@ -5,8 +5,10 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}; +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}, +}; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/create_experiment.rs b/dask_planner/src/sql/logical/create_experiment.rs index e36c31983..313357d75 100644 --- a/dask_planner/src/sql/logical/create_experiment.rs +++ b/dask_planner/src/sql/logical/create_experiment.rs @@ -5,8 +5,10 @@ use std::{ sync::Arc, }; -use datafusion_common::DFSchemaRef; -use datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}; +use datafusion_python::{ + datafusion_common::DFSchemaRef, + datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}, +}; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/create_memory_table.rs b/dask_planner/src/sql/logical/create_memory_table.rs index 9a8dafc75..668295e0f 100644 --- a/dask_planner/src/sql/logical/create_memory_table.rs +++ b/dask_planner/src/sql/logical/create_memory_table.rs @@ -1,4 +1,4 @@ -use datafusion_expr::{ +use datafusion_python::datafusion_expr::{ logical_plan::{CreateMemoryTable, CreateView}, LogicalPlan, }; diff --git a/dask_planner/src/sql/logical/create_model.rs b/dask_planner/src/sql/logical/create_model.rs index eeeacbefa..782fe3325 100644 --- a/dask_planner/src/sql/logical/create_model.rs +++ b/dask_planner/src/sql/logical/create_model.rs @@ -5,8 +5,10 @@ use std::{ sync::Arc, }; -use datafusion_common::DFSchemaRef; -use datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}; +use datafusion_python::{ + datafusion_common::DFSchemaRef, + datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}, +}; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/create_table.rs b/dask_planner/src/sql/logical/create_table.rs index 7be91a3e2..9271130c7 100644 --- a/dask_planner/src/sql/logical/create_table.rs +++ b/dask_planner/src/sql/logical/create_table.rs @@ -5,8 +5,10 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}; +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}, +}; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/describe_model.rs b/dask_planner/src/sql/logical/describe_model.rs index b9cad11b9..cb2087376 100644 --- a/dask_planner/src/sql/logical/describe_model.rs +++ b/dask_planner/src/sql/logical/describe_model.rs @@ -5,8 +5,10 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}; +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}, +}; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/drop_model.rs b/dask_planner/src/sql/logical/drop_model.rs index 733aeeedc..71074905d 100644 --- a/dask_planner/src/sql/logical/drop_model.rs +++ b/dask_planner/src/sql/logical/drop_model.rs @@ -5,8 +5,10 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}; +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}, +}; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/drop_schema.rs b/dask_planner/src/sql/logical/drop_schema.rs index 31d9386f0..2022a61c9 100644 --- a/dask_planner/src/sql/logical/drop_schema.rs +++ b/dask_planner/src/sql/logical/drop_schema.rs @@ -5,8 +5,10 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}; +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}, +}; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/drop_table.rs b/dask_planner/src/sql/logical/drop_table.rs index d55c0d243..7d58e8a47 100644 --- a/dask_planner/src/sql/logical/drop_table.rs +++ b/dask_planner/src/sql/logical/drop_table.rs @@ -1,4 +1,4 @@ -use datafusion_expr::logical_plan::{DropTable, LogicalPlan}; +use datafusion_python::datafusion_expr::logical_plan::{DropTable, LogicalPlan}; use pyo3::prelude::*; use crate::sql::exceptions::py_type_err; diff --git a/dask_planner/src/sql/logical/empty_relation.rs b/dask_planner/src/sql/logical/empty_relation.rs index 864cb18ea..5bd6659ce 100644 --- a/dask_planner/src/sql/logical/empty_relation.rs +++ b/dask_planner/src/sql/logical/empty_relation.rs @@ -1,4 +1,4 @@ -use datafusion_expr::logical_plan::{EmptyRelation, LogicalPlan}; +use datafusion_python::datafusion_expr::logical_plan::{EmptyRelation, LogicalPlan}; use pyo3::prelude::*; use crate::sql::exceptions::py_type_err; diff --git a/dask_planner/src/sql/logical/explain.rs b/dask_planner/src/sql/logical/explain.rs index a23c4ea4f..17f1e4ee2 100644 --- a/dask_planner/src/sql/logical/explain.rs +++ b/dask_planner/src/sql/logical/explain.rs @@ -1,4 +1,4 @@ -use datafusion_expr::{logical_plan::Explain, LogicalPlan}; +use datafusion_python::datafusion_expr::{logical_plan::Explain, LogicalPlan}; use pyo3::prelude::*; use crate::sql::exceptions::py_type_err; diff --git a/dask_planner/src/sql/logical/export_model.rs b/dask_planner/src/sql/logical/export_model.rs index 9a4535674..e38551b58 100644 --- a/dask_planner/src/sql/logical/export_model.rs +++ b/dask_planner/src/sql/logical/export_model.rs @@ -5,8 +5,10 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}; +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}, +}; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/filter.rs b/dask_planner/src/sql/logical/filter.rs index 532d98374..a50d508ff 100644 --- a/dask_planner/src/sql/logical/filter.rs +++ b/dask_planner/src/sql/logical/filter.rs @@ -1,4 +1,4 @@ -use datafusion_expr::{logical_plan::Filter, LogicalPlan}; +use datafusion_python::datafusion_expr::{logical_plan::Filter, LogicalPlan}; use pyo3::prelude::*; use crate::{expression::PyExpr, sql::exceptions::py_type_err}; diff --git a/dask_planner/src/sql/logical/join.rs b/dask_planner/src/sql/logical/join.rs index 10a45a6e5..d6c31b55b 100644 --- a/dask_planner/src/sql/logical/join.rs +++ b/dask_planner/src/sql/logical/join.rs @@ -1,10 +1,12 @@ -use datafusion_common::Column; -use datafusion_expr::{ - and, - logical_plan::{Join, JoinType, LogicalPlan}, - BinaryExpr, - Expr, - Operator, +use datafusion_python::{ + datafusion_common::Column, + datafusion_expr::{ + and, + logical_plan::{Join, JoinType, LogicalPlan}, + BinaryExpr, + Expr, + Operator, + }, }; use pyo3::prelude::*; @@ -70,30 +72,28 @@ impl PyJoin { #[pyo3(name = "getJoinConditions")] pub fn join_conditions(&mut self) -> PyResult> { - let lhs_table_name = match &*self.join.left { - LogicalPlan::TableScan(scan) => scan.table_name.clone(), - _ => { - return Err(py_type_err( - "lhs Expected TableScan but something else was received!", - )) - } - }; + // let lhs_table_name = match &*self.join.left { + // LogicalPlan::TableScan(scan) => scan.table_name.clone(), + // _ => { + // return Err(py_type_err( + // "lhs Expected TableScan but something else was received!", + // )) + // } + // }; - let rhs_table_name = match &*self.join.right { - LogicalPlan::TableScan(scan) => scan.table_name.clone(), - _ => { - return Err(py_type_err( - "rhs Expected TableScan but something else was received!", - )) - } - }; + // let rhs_table_name = match &*self.join.right { + // LogicalPlan::TableScan(scan) => scan.table_name.clone(), + // _ => { + // return Err(py_type_err( + // "rhs Expected TableScan but something else was received!", + // )) + // } + // }; let mut join_conditions: Vec<(column::PyColumn, column::PyColumn)> = Vec::new(); for (lhs, rhs) in self.join.on.clone() { match (lhs, rhs) { - (Expr::Column(mut lhs), Expr::Column(mut rhs)) => { - lhs.relation = Some(lhs_table_name.clone()); - rhs.relation = Some(rhs_table_name.clone()); + (Expr::Column(lhs), Expr::Column(rhs)) => { join_conditions.push((lhs.into(), rhs.into())); } _ => return Err(py_type_err("unsupported join condition")), diff --git a/dask_planner/src/sql/logical/limit.rs b/dask_planner/src/sql/logical/limit.rs index 750bbe834..189fdeea0 100644 --- a/dask_planner/src/sql/logical/limit.rs +++ b/dask_planner/src/sql/logical/limit.rs @@ -1,5 +1,7 @@ -use datafusion_common::ScalarValue; -use datafusion_expr::{logical_plan::Limit, Expr, LogicalPlan}; +use datafusion_python::{ + datafusion_common::ScalarValue, + datafusion_expr::{logical_plan::Limit, Expr, LogicalPlan}, +}; use pyo3::prelude::*; use crate::{expression::PyExpr, sql::exceptions::py_type_err}; diff --git a/dask_planner/src/sql/logical/predict_model.rs b/dask_planner/src/sql/logical/predict_model.rs index 6c4e358e4..e8d723d2c 100644 --- a/dask_planner/src/sql/logical/predict_model.rs +++ b/dask_planner/src/sql/logical/predict_model.rs @@ -5,8 +5,10 @@ use std::{ sync::Arc, }; -use datafusion_common::DFSchemaRef; -use datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}; +use datafusion_python::{ + datafusion_common::DFSchemaRef, + datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}, +}; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/projection.rs b/dask_planner/src/sql/logical/projection.rs index a017731b5..99ed0d684 100644 --- a/dask_planner/src/sql/logical/projection.rs +++ b/dask_planner/src/sql/logical/projection.rs @@ -1,4 +1,4 @@ -use datafusion_expr::{logical_plan::Projection, Expr, LogicalPlan}; +use datafusion_python::datafusion_expr::{logical_plan::Projection, Expr, LogicalPlan}; use pyo3::prelude::*; use crate::{expression::PyExpr, sql::exceptions::py_type_err}; @@ -34,8 +34,16 @@ impl PyProjection { let py_expr: PyExpr = PyExpr::from(expression, Some(vec![self.projection.input.clone()])); for expr in self.projected_expressions(&py_expr) { - if let Ok(name) = expr._column_name(&self.projection.input) { - named.push((name, expr.clone())); + match expr.expr { + Expr::Alias(ex, name) => named.push(( + name.to_string(), + PyExpr::from(*ex, Some(vec![self.projection.input.clone()])), + )), + _ => { + if let Ok(name) = expr._column_name(&self.projection.input) { + named.push((name, expr.clone())); + } + } } } } diff --git a/dask_planner/src/sql/logical/repartition_by.rs b/dask_planner/src/sql/logical/repartition_by.rs index c5f77d53d..e931b88e7 100644 --- a/dask_planner/src/sql/logical/repartition_by.rs +++ b/dask_planner/src/sql/logical/repartition_by.rs @@ -1,4 +1,4 @@ -use datafusion_expr::{ +use datafusion_python::datafusion_expr::{ logical_plan::{Partitioning, Repartition}, Expr, LogicalPlan, diff --git a/dask_planner/src/sql/logical/show_columns.rs b/dask_planner/src/sql/logical/show_columns.rs index 6ae27dc36..adfb584ef 100644 --- a/dask_planner/src/sql/logical/show_columns.rs +++ b/dask_planner/src/sql/logical/show_columns.rs @@ -5,11 +5,13 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{ - logical_plan::{Extension, UserDefinedLogicalNode}, - Expr, - LogicalPlan, +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{ + logical_plan::{Extension, UserDefinedLogicalNode}, + Expr, + LogicalPlan, + }, }; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/show_models.rs b/dask_planner/src/sql/logical/show_models.rs index b5ba50264..026a179a5 100644 --- a/dask_planner/src/sql/logical/show_models.rs +++ b/dask_planner/src/sql/logical/show_models.rs @@ -5,8 +5,10 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}; +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}, +}; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/show_schemas.rs b/dask_planner/src/sql/logical/show_schemas.rs index 732cda7c1..3e3ed4783 100644 --- a/dask_planner/src/sql/logical/show_schemas.rs +++ b/dask_planner/src/sql/logical/show_schemas.rs @@ -5,11 +5,13 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{ - logical_plan::{Extension, UserDefinedLogicalNode}, - Expr, - LogicalPlan, +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{ + logical_plan::{Extension, UserDefinedLogicalNode}, + Expr, + LogicalPlan, + }, }; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/show_tables.rs b/dask_planner/src/sql/logical/show_tables.rs index 023c19063..987f2546e 100644 --- a/dask_planner/src/sql/logical/show_tables.rs +++ b/dask_planner/src/sql/logical/show_tables.rs @@ -5,11 +5,13 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{ - logical_plan::{Extension, UserDefinedLogicalNode}, - Expr, - LogicalPlan, +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{ + logical_plan::{Extension, UserDefinedLogicalNode}, + Expr, + LogicalPlan, + }, }; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/sort.rs b/dask_planner/src/sql/logical/sort.rs index 06d35a28f..9abcd3906 100644 --- a/dask_planner/src/sql/logical/sort.rs +++ b/dask_planner/src/sql/logical/sort.rs @@ -1,4 +1,4 @@ -use datafusion_expr::{logical_plan::Sort, LogicalPlan}; +use datafusion_python::datafusion_expr::{logical_plan::Sort, LogicalPlan}; use pyo3::prelude::*; use crate::{ diff --git a/dask_planner/src/sql/logical/subquery_alias.rs b/dask_planner/src/sql/logical/subquery_alias.rs index 12ff5b3c1..1b23e5dc4 100644 --- a/dask_planner/src/sql/logical/subquery_alias.rs +++ b/dask_planner/src/sql/logical/subquery_alias.rs @@ -1,4 +1,4 @@ -use datafusion_expr::{logical_plan::SubqueryAlias, LogicalPlan}; +use datafusion_python::datafusion_expr::{logical_plan::SubqueryAlias, LogicalPlan}; use pyo3::prelude::*; use crate::sql::exceptions::py_type_err; diff --git a/dask_planner/src/sql/logical/table_scan.rs b/dask_planner/src/sql/logical/table_scan.rs index 537f011cc..7f9253bcd 100644 --- a/dask_planner/src/sql/logical/table_scan.rs +++ b/dask_planner/src/sql/logical/table_scan.rs @@ -1,7 +1,9 @@ use std::sync::Arc; -use datafusion_common::DFSchema; -use datafusion_expr::{logical_plan::TableScan, LogicalPlan}; +use datafusion_python::{ + datafusion_common::DFSchema, + datafusion_expr::{logical_plan::TableScan, LogicalPlan}, +}; use pyo3::prelude::*; use crate::{ diff --git a/dask_planner/src/sql/logical/use_schema.rs b/dask_planner/src/sql/logical/use_schema.rs index ee974c8cb..7c2206310 100644 --- a/dask_planner/src/sql/logical/use_schema.rs +++ b/dask_planner/src/sql/logical/use_schema.rs @@ -5,8 +5,10 @@ use std::{ sync::Arc, }; -use datafusion_common::{DFSchema, DFSchemaRef}; -use datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}; +use datafusion_python::{ + datafusion_common::{DFSchema, DFSchemaRef}, + datafusion_expr::{logical_plan::UserDefinedLogicalNode, Expr, LogicalPlan}, +}; use fmt::Debug; use pyo3::prelude::*; diff --git a/dask_planner/src/sql/logical/window.rs b/dask_planner/src/sql/logical/window.rs index a9c83b1ad..e104ccdb3 100644 --- a/dask_planner/src/sql/logical/window.rs +++ b/dask_planner/src/sql/logical/window.rs @@ -1,11 +1,13 @@ -use datafusion_common::ScalarValue; -use datafusion_expr::{ - expr::WindowFunction, - logical_plan::Window, - Expr, - LogicalPlan, - WindowFrame, - WindowFrameBound, +use datafusion_python::{ + datafusion_common::ScalarValue, + datafusion_expr::{ + expr::WindowFunction, + logical_plan::Window, + Expr, + LogicalPlan, + WindowFrame, + WindowFrameBound, + }, }; use pyo3::prelude::*; @@ -171,6 +173,16 @@ impl PyWindowFrameBound { ScalarValue::UInt64(v) => Ok(*v), // The cast below is only safe because window bounds cannot be negative ScalarValue::Int64(v) => Ok(v.map(|n| n as u64)), + ScalarValue::Utf8(v) => { + let s = v.clone().unwrap(); + match s.parse::() { + Ok(s) => Ok(Some(s)), + Err(_e) => Err(DaskPlannerError::Internal(format!( + "Unable to parse u64 from Utf8 value '{s}'" + )) + .into()), + } + } ref x => Err(DaskPlannerError::Internal(format!( "Unexpected window frame bound: {x}" )) diff --git a/dask_planner/src/sql/optimizer.rs b/dask_planner/src/sql/optimizer.rs index 8950a9334..68577cf2c 100644 --- a/dask_planner/src/sql/optimizer.rs +++ b/dask_planner/src/sql/optimizer.rs @@ -1,33 +1,26 @@ use std::sync::Arc; -use datafusion_common::DataFusionError; -use datafusion_expr::LogicalPlan; -use datafusion_optimizer::{ - // common_subexpr_eliminate::CommonSubexprEliminate, - decorrelate_where_exists::DecorrelateWhereExists, - decorrelate_where_in::DecorrelateWhereIn, - eliminate_cross_join::EliminateCrossJoin, - // eliminate_duplicated_expr::EliminateDuplicatedExpr, - // eliminate_filter::EliminateFilter, - eliminate_limit::EliminateLimit, - eliminate_outer_join::EliminateOuterJoin, - eliminate_project::EliminateProjection, - // extract_equijoin_predicate::ExtractEquijoinPredicate, - filter_null_join_keys::FilterNullJoinKeys, - // merge_projection::MergeProjection, - optimizer::{Optimizer, OptimizerRule}, - // propagate_empty_relation::PropagateEmptyRelation, - push_down_filter::PushDownFilter, - push_down_limit::PushDownLimit, - push_down_projection::PushDownProjection, - // replace_distinct_aggregate::ReplaceDistinctWithAggregate, - rewrite_disjunctive_predicate::RewriteDisjunctivePredicate, - scalar_subquery_to_join::ScalarSubqueryToJoin, - simplify_expressions::SimplifyExpressions, - // single_distinct_to_groupby::SingleDistinctToGroupBy, - type_coercion::TypeCoercion, - unwrap_cast_in_comparison::UnwrapCastInComparison, - OptimizerContext, +use datafusion_python::{ + datafusion_common::DataFusionError, + datafusion_expr::LogicalPlan, + datafusion_optimizer::{ + decorrelate_where_exists::DecorrelateWhereExists, + decorrelate_where_in::DecorrelateWhereIn, + eliminate_cross_join::EliminateCrossJoin, + eliminate_limit::EliminateLimit, + eliminate_outer_join::EliminateOuterJoin, + eliminate_project::EliminateProjection, + filter_null_join_keys::FilterNullJoinKeys, + optimizer::{Optimizer, OptimizerRule}, + push_down_filter::PushDownFilter, + push_down_limit::PushDownLimit, + push_down_projection::PushDownProjection, + rewrite_disjunctive_predicate::RewriteDisjunctivePredicate, + scalar_subquery_to_join::ScalarSubqueryToJoin, + simplify_expressions::SimplifyExpressions, + unwrap_cast_in_comparison::UnwrapCastInComparison, + OptimizerContext, + }, }; use log::{debug, trace}; @@ -45,8 +38,8 @@ impl DaskSqlOptimizer { /// optimizers as well as any custom `OptimizerRule` trait impls that might be desired. pub fn new() -> Self { debug!("Creating new instance of DaskSqlOptimizer"); + let rules: Vec> = vec![ - Arc::new(TypeCoercion::new()), Arc::new(SimplifyExpressions::new()), Arc::new(UnwrapCastInComparison::new()), // Arc::new(ReplaceDistinctWithAggregate::new()), @@ -113,13 +106,15 @@ impl DaskSqlOptimizer { mod tests { use std::{any::Any, collections::HashMap, sync::Arc}; - use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; - use datafusion_common::{config::ConfigOptions, DataFusionError, Result}; - use datafusion_expr::{AggregateUDF, LogicalPlan, ScalarUDF, TableSource}; - use datafusion_sql::{ - planner::{ContextProvider, SqlToRel}, - sqlparser::{ast::Statement, parser::Parser}, - TableReference, + use datafusion_python::{ + datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}, + datafusion_common::{config::ConfigOptions, DataFusionError, Result}, + datafusion_expr::{AggregateUDF, LogicalPlan, ScalarUDF, TableSource}, + datafusion_sql::{ + planner::{ContextProvider, SqlToRel}, + sqlparser::{ast::Statement, parser::Parser}, + TableReference, + }, }; use crate::{dialect::DaskDialect, sql::optimizer::DaskSqlOptimizer}; @@ -184,7 +179,7 @@ mod tests { fn get_table_provider( &self, name: TableReference, - ) -> datafusion_common::Result> { + ) -> datafusion_python::datafusion_common::Result> { let table_name = name.table(); if table_name.starts_with("test") { let schema = Schema::new_with_metadata( diff --git a/dask_planner/src/sql/optimizer/join_reorder.rs b/dask_planner/src/sql/optimizer/join_reorder.rs index d189aba7e..8997f9d96 100644 --- a/dask_planner/src/sql/optimizer/join_reorder.rs +++ b/dask_planner/src/sql/optimizer/join_reorder.rs @@ -3,9 +3,11 @@ use std::collections::HashSet; -use datafusion_common::{Column, Result}; -use datafusion_expr::{Expr, Join, JoinType, LogicalPlan, LogicalPlanBuilder}; -use datafusion_optimizer::{utils, utils::split_conjunction, OptimizerConfig, OptimizerRule}; +use datafusion_python::{ + datafusion_common::{Column, Result}, + datafusion_expr::{Expr, Join, JoinType, LogicalPlan, LogicalPlanBuilder}, + datafusion_optimizer::{utils, utils::split_conjunction, OptimizerConfig, OptimizerRule}, +}; use log::warn; use crate::sql::table::DaskTableSource; diff --git a/dask_planner/src/sql/parser_utils.rs b/dask_planner/src/sql/parser_utils.rs index b28f0ad70..a6a6c8fac 100644 --- a/dask_planner/src/sql/parser_utils.rs +++ b/dask_planner/src/sql/parser_utils.rs @@ -1,4 +1,4 @@ -use datafusion_sql::sqlparser::{ast::ObjectName, parser::ParserError}; +use datafusion_python::datafusion_sql::sqlparser::{ast::ObjectName, parser::ParserError}; pub struct DaskParserUtils; diff --git a/dask_planner/src/sql/table.rs b/dask_planner/src/sql/table.rs index f105fee48..f25f891ec 100644 --- a/dask_planner/src/sql/table.rs +++ b/dask_planner/src/sql/table.rs @@ -1,11 +1,13 @@ use std::{any::Any, sync::Arc}; use async_trait::async_trait; -use datafusion::arrow::datatypes::{DataType, Field, SchemaRef}; -use datafusion_common::DFField; -use datafusion_expr::{Expr, LogicalPlan, TableProviderFilterPushDown, TableSource}; -use datafusion_optimizer::utils::split_conjunction; -use datafusion_sql::TableReference; +use datafusion_python::{ + datafusion::arrow::datatypes::{DataType, Field, SchemaRef}, + datafusion_common::DFField, + datafusion_expr::{Expr, LogicalPlan, TableProviderFilterPushDown, TableSource}, + datafusion_optimizer::utils::split_conjunction, + datafusion_sql::TableReference, +}; use pyo3::prelude::*; use super::logical::{create_table::CreateTablePlanNode, predict_model::PredictModelPlanNode}; @@ -69,7 +71,7 @@ impl TableSource for DaskTableSource { fn supports_filter_pushdown( &self, filter: &Expr, - ) -> datafusion_common::Result { + ) -> datafusion_python::datafusion_common::Result { let filters = split_conjunction(filter); if filters.iter().all(|f| is_supported_push_down_expr(f)) { // Push down filters to the tablescan operation if all are supported diff --git a/dask_planner/src/sql/types.rs b/dask_planner/src/sql/types.rs index d316abd48..ceff904a6 100644 --- a/dask_planner/src/sql/types.rs +++ b/dask_planner/src/sql/types.rs @@ -1,8 +1,10 @@ pub mod rel_data_type; pub mod rel_data_type_field; -use datafusion::arrow::datatypes::{DataType, IntervalUnit, TimeUnit}; -use datafusion_sql::sqlparser::{ast::DataType as SQLType, parser::Parser, tokenizer::Tokenizer}; +use datafusion_python::{ + datafusion::arrow::datatypes::{DataType, IntervalUnit, TimeUnit}, + datafusion_sql::sqlparser::{ast::DataType as SQLType, parser::Parser, tokenizer::Tokenizer}, +}; use pyo3::{prelude::*, types::PyDict}; use crate::{dialect::DaskDialect, error::DaskPlannerError, sql::exceptions::py_type_err}; @@ -231,6 +233,7 @@ pub enum SqlTypeName { INTERVAL_MINUTE, INTERVAL_MINUTE_SECOND, INTERVAL_MONTH, + INTERVAL_MONTH_DAY_NANOSECOND, INTERVAL_SECOND, INTERVAL_YEAR, INTERVAL_YEAR_MONTH, @@ -299,7 +302,7 @@ impl SqlTypeName { DataType::Interval(unit) => match unit { IntervalUnit::DayTime => Ok(SqlTypeName::INTERVAL_DAY), IntervalUnit::YearMonth => Ok(SqlTypeName::INTERVAL_YEAR_MONTH), - IntervalUnit::MonthDayNano => Ok(SqlTypeName::INTERVAL_MONTH), + IntervalUnit::MonthDayNano => Ok(SqlTypeName::INTERVAL_MONTH_DAY_NANOSECOND), }, DataType::Binary => Ok(SqlTypeName::BINARY), DataType::FixedSizeBinary(_size) => Ok(SqlTypeName::VARBINARY), diff --git a/dask_planner/src/sql/types/rel_data_type_field.rs b/dask_planner/src/sql/types/rel_data_type_field.rs index bac827abd..13f036d0e 100644 --- a/dask_planner/src/sql/types/rel_data_type_field.rs +++ b/dask_planner/src/sql/types/rel_data_type_field.rs @@ -1,7 +1,9 @@ use std::fmt; -use datafusion_common::{DFField, DFSchema}; -use datafusion_sql::TableReference; +use datafusion_python::{ + datafusion_common::{DFField, DFSchema}, + datafusion_sql::TableReference, +}; use pyo3::prelude::*; use crate::{ diff --git a/dask_planner/update-dependencies.sh b/dask_planner/update-dependencies.sh index a94c53dad..9fff256f6 100644 --- a/dask_planner/update-dependencies.sh +++ b/dask_planner/update-dependencies.sh @@ -2,7 +2,7 @@ UPDATE_ALL_CARGO_DEPS="${UPDATE_ALL_CARGO_DEPS:-true}" # Update datafusion dependencies in the dask-planner to the latest revision from the default branch -sed -i -r 's/^datafusion-([a-z]+).*/datafusion-\1 = { git = "https:\/\/github.com\/apache\/arrow-datafusion\/" }/g' Cargo.toml +sed -i -r 's/^datafusion-([a-z]+).*/datafusion-\1 = { git = "https:\/\/github.com\/apache\/arrow-datafusion-python\/" }/g' Cargo.toml if [ "$UPDATE_ALL_CARGO_DEPS" = true ] ; then cargo update diff --git a/dask_sql/mappings.py b/dask_sql/mappings.py index 6b9a8154f..a5a14c13f 100644 --- a/dask_sql/mappings.py +++ b/dask_sql/mappings.py @@ -89,6 +89,7 @@ unit="ns", tz="UTC" ), # Everything is converted to UTC. So far, this did not break "SqlTypeName.INTERVAL_DAY": np.dtype(" Any: # Calcite will always convert INTERVAL types except YEAR, QUATER, MONTH to milliseconds # Issue: if sql_type is INTERVAL MICROSECOND, and value <= 1000, literal_value will be rounded to 0 return np.timedelta64(literal_value, "ms") + elif sql_type == SqlTypeName.INTERVAL_MONTH_DAY_NANOSECOND: + # DataFusion assumes 30 days per month. Therefore we multiply number of months by 30 and add to days + return np.timedelta64( + (literal_value[0] * 30) + literal_value[1], "D" + ) + np.timedelta64(literal_value[2], "ns") elif sql_type == SqlTypeName.BOOLEAN: return bool(literal_value) diff --git a/dask_sql/physical/rex/core/literal.py b/dask_sql/physical/rex/core/literal.py index 482484eee..73e3b8185 100644 --- a/dask_sql/physical/rex/core/literal.py +++ b/dask_sql/physical/rex/core/literal.py @@ -161,6 +161,9 @@ def convert( elif literal_type == "IntervalDayTime": literal_type = SqlTypeName.INTERVAL_DAY literal_value = rex.getIntervalDayTimeValue() + elif literal_type == "IntervalMonthDayNano": + literal_type = SqlTypeName.INTERVAL_MONTH_DAY_NANOSECOND + literal_value = rex.getIntervalMonthDayNanoValue() elif literal_type in { "TimestampSecond", "TimestampMillisecond", diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..dfed2ba50 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["setuptools", "wheel", "setuptools-rust"] + +[tool.isort] +profile = "black" + +[tool.maturin] +include = [ + { path = "Cargo.lock", format = "sdist" } +] +exclude = [".github/**", "ci/**", ".asf.yaml"] +# Require Cargo.lock is up to date +locked = true