Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] updates for delta-rs #189

Draft
wants to merge 16 commits into
base: main
Choose a base branch
from
Draft

[WIP] updates for delta-rs #189

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
11a8748
chore: bump arrow
roeap May 2, 2024
fae0743
feat: add serde to some types
roeap May 2, 2024
9c8f7f5
chore: more serde config
roeap May 4, 2024
891cecb
Merge branch 'main' into feature/delta-rs
roeap May 8, 2024
4a9945a
chore: bump object store
roeap May 8, 2024
87ecd87
fix: inspect-table example
roeap May 8, 2024
af9f65b
fix: some data types
roeap May 8, 2024
49295c7
fix: refernce datatype in comparison
roeap May 8, 2024
e10d011
style: better error message
roeap May 8, 2024
3829c83
feat: add reader and writer feature enums
roeap May 9, 2024
005e521
feat: extend MEtadata Value
roeap May 9, 2024
796be6d
feat: iterator convenince functions for StructType
roeap May 9, 2024
17bbd63
feat: read physical name from metadata
roeap May 9, 2024
524262a
feat: extend scalar and fix more ts
roeap May 9, 2024
312044f
fix: add dictitionaly type conversion
roeap May 9, 2024
6eff5e2
fix: better deciaml handling
roeap May 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 1 addition & 8 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,5 @@
[workspace]
members = [
"acceptance",
"derive-macros",
"ffi",
"kernel",
"kernel/examples/dump-table", # todo: put back to `examples/*` when inspect-table is fixed
"kernel/examples/read-table-multi-threaded"
]
members = ["acceptance", "derive-macros", "ffi", "kernel", "kernel/examples/*"]
# Only check / build main crates by default (check all with `--workspace`)
default-members = ["acceptance", "kernel"]
resolver = "2"
Expand Down
17 changes: 10 additions & 7 deletions acceptance/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,17 @@ readme.workspace = true
version.workspace = true

[dependencies]
arrow-array = { version = "^49.0" }
arrow-ord = { version = "^49.0" }
arrow-select = { version = "^49.0" }
arrow-schema = { version = "^49.0" }
delta_kernel = { path = "../kernel", features = ["default-engine", "developer-visibility"] }
arrow-array = { version = "^51.0" }
arrow-ord = { version = "^51.0" }
arrow-select = { version = "^51.0" }
arrow-schema = { version = "^51.0" }
delta_kernel = { path = "../kernel", features = [
"default-engine",
"developer-visibility",
] }
futures = "0.3"
object_store = "^0.8.0"
parquet = { version = "^49.0" , features = ["object_store"] }
object_store = "^0.9.0"
parquet = { version = "^51.0", features = ["object_store"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
thiserror = "1"
Expand Down
16 changes: 11 additions & 5 deletions ffi/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,18 @@ crate-type = ["cdylib", "staticlib"]
[dependencies]
tracing = "0.1"
url = "2"
delta_kernel = { path = "../kernel", default-features = false, features = ["developer-visibility"] }
delta_kernel = { path = "../kernel", default-features = false, features = [
"developer-visibility",
] }

# used if we use the default engine to be able to move arrow data into the c-ffi format
arrow-schema = { version = "^49.0", default-features = false, features = ["ffi"], optional = true }
arrow-data = { version = "^49.0", default-features = false, features = ["ffi"], optional = true }
arrow-array = { version = "^49.0", default-features = false, optional = true }
arrow-schema = { version = "^51.0", default-features = false, features = [
"ffi",
], optional = true }
arrow-data = { version = "^51.0", default-features = false, features = [
"ffi",
], optional = true }
arrow-array = { version = "^51.0", default-features = false, optional = true }

[build-dependencies]
cbindgen = "0.26.0"
Expand All @@ -32,6 +38,6 @@ default-engine = [
"delta_kernel/default-engine",
"arrow-array",
"arrow-data",
"arrow-schema"
"arrow-schema",
]
sync-engine = ["delta_kernel/sync-engine"]
21 changes: 11 additions & 10 deletions kernel/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,16 +37,16 @@ delta_kernel_derive = { path = "../derive-macros", version = "0.0.1" }
visibility = "0.1.0"

# Used in default engine
arrow-array = { version = "^49.0", optional = true }
arrow-select = { version = "^49.0", optional = true }
arrow-arith = { version = "^49.0", optional = true }
arrow-json = { version = "^49.0", optional = true }
arrow-ord = { version = "^49.0", optional = true }
arrow-schema = { version = "^49.0", optional = true }
arrow-array = { version = "^51.0", optional = true }
arrow-select = { version = "^51.0", optional = true }
arrow-arith = { version = "^51.0", optional = true }
arrow-json = { version = "^51.0", optional = true }
arrow-ord = { version = "^51.0", optional = true }
arrow-schema = { version = "^51.0", optional = true }
futures = { version = "0.3", optional = true }
object_store = { version = "^0.8.0", optional = true }
object_store = { version = "^0.9.0", optional = true }
# Used in default and sync engine
parquet = { version = "^49.0", optional = true }
parquet = { version = "^51.0", optional = true }
# Used for fetching direct urls (like pre-signed urls)
reqwest = { version = "^0.11.0", optional = true }

Expand All @@ -71,6 +71,7 @@ default-engine = [
"reqwest",
"tokio",
]
serde = []

developer-visibility = []
sync-engine = [
Expand All @@ -79,14 +80,14 @@ sync-engine = [
"arrow-array",
"arrow-json",
"arrow-select",
"parquet"
"parquet",
]

[build-dependencies]
rustc_version = "0.4.0"

[dev-dependencies]
arrow = { version = "^49.0", features = ["json", "prettyprint"] }
arrow = { version = "^51.0", features = ["json", "prettyprint"] }
delta_kernel = { path = ".", features = ["default-engine", "sync-engine"] }
test-log = { version = "0.2", default-features = false, features = ["trace"] }
tempfile = "3"
Expand Down
3 changes: 1 addition & 2 deletions kernel/examples/dump-table/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ edition = "2021"
publish = false

[dependencies]
arrow = { version = "^49.0", features = ["prettyprint"] }
arrow = { version = "^51.0", features = ["prettyprint"] }
clap = { version = "^4.4", features = ["derive"] }
delta_kernel = { path = "../../../kernel", features = [
"default-engine",
Expand All @@ -15,4 +15,3 @@ delta_kernel = { path = "../../../kernel", features = [
env_logger = "0.11.3"
itertools = "0.12.1"
url = "2"

9 changes: 5 additions & 4 deletions kernel/examples/inspect-table/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ edition = "2021"
publish = false

[dependencies]
arrow-array = { version = "^49.0" }
arrow-schema = { version = "^49.0" }
arrow-array = { version = "^51.0" }
arrow-schema = { version = "^51.0" }
clap = { version = "^4.4", features = ["derive"] }
delta_kernel = { path = "../../../kernel", features = [
"default",
"default-engine",
"tokio",
"developer-visibility",
] }
clap = { version = "^4.4", features = ["derive"] }
env_logger = "0.11.3"
url = "2"
Loading
Loading