diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee7fa799b96..cba6064e6c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,7 @@ jobs: - test --profile=ci-dev -p cairo-lang-test-utils - test --profile=ci-dev -p cairo-lang-utils --features=serde,parity-scale-codec,schemars,testing,env_logger - test --profile=ci-dev -p cairo-lang-utils --no-default-features --features=serde,parity-scale-codec + - test --profile=ci-dev -p cairo-lang-primitive-token - test --profile=ci-dev -p tests steps: - uses: actions/checkout@v4 diff --git a/Cargo.lock b/Cargo.lock index e311914a857..ddd7bf0c2e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -831,6 +831,10 @@ dependencies = [ "test-log", ] +[[package]] +name = "cairo-lang-primitive-token" +version = "1.0.0" + [[package]] name = "cairo-lang-proc-macros" version = "2.9.0" @@ -1128,6 +1132,7 @@ version = "2.9.0" dependencies = [ "cairo-lang-debug", "cairo-lang-filesystem", + "cairo-lang-primitive-token", "cairo-lang-utils", "env_logger", "num-bigint", diff --git a/Cargo.toml b/Cargo.toml index 379e688a471..8ab8167b0e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,7 @@ members = [ "crates/cairo-lang-sierra-generator", "crates/cairo-lang-sierra-to-casm", "crates/cairo-lang-sierra-type-size", + "crates/cairo-lang-primitive-token", "crates/cairo-lang-starknet", "crates/cairo-lang-starknet-classes", "crates/cairo-lang-syntax", diff --git a/crates/cairo-lang-primitive-token/Cargo.toml b/crates/cairo-lang-primitive-token/Cargo.toml new file mode 100644 index 00000000000..871cec11b50 --- /dev/null +++ b/crates/cairo-lang-primitive-token/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "cairo-lang-primitive-token" +version = "1.0.0" +edition.workspace = true +repository.workspace = true +license-file.workspace = true +description = "Primitive representation of a TokenStream" diff --git a/crates/cairo-lang-primitive-token/src/lib.rs b/crates/cairo-lang-primitive-token/src/lib.rs new file mode 100644 index 00000000000..3313004d392 --- /dev/null +++ b/crates/cairo-lang-primitive-token/src/lib.rs @@ -0,0 +1,35 @@ +#![deny(missing_docs)] +//! This crate defines unfiorm and primitive form of the TokenStream. +//! We want this to be as stable as possible and limit the changes here to bare minimum. + +/// Primitive representation of a token's span. +pub struct PrimitiveSpan { + /// Start position of the span. + pub start: usize, + /// End position of the span. + pub end: usize, +} + +/// Primitive representation of a single token. +pub struct PrimitiveToken { + /// Plain code content that the token represents (includes whitespaces). + pub content: String, + /// Span of the token. + pub span: Option, +} + +impl PrimitiveToken { + /// Creates a new primitive token based upon content and provided span. + pub fn new(content: String, span: Option) -> Self { + Self { content, span } + } +} + +/// Trait that defines an object that can be turned into a PrimitiveTokenStream iterator. +pub trait ToPrimitiveTokenStream { + /// Iterator type for PrimitiveTokens. + type Iter: Iterator; + + /// Method that turns given item to a PrimitiveTokenStream iterator. + fn to_primitive_token_stream(&self) -> Self::Iter; +} diff --git a/crates/cairo-lang-syntax/Cargo.toml b/crates/cairo-lang-syntax/Cargo.toml index 042ee4eb326..a944a433d9a 100644 --- a/crates/cairo-lang-syntax/Cargo.toml +++ b/crates/cairo-lang-syntax/Cargo.toml @@ -10,6 +10,7 @@ description = "Cairo syntax representation." cairo-lang-debug = { path = "../cairo-lang-debug", version = "~2.9.0" } cairo-lang-filesystem = { path = "../cairo-lang-filesystem", version = "~2.9.0" } cairo-lang-utils = { path = "../cairo-lang-utils", version = "~2.9.0" } +cairo-lang-primitive-token = { path = "../cairo-lang-primitive-token", version = "1.0.0"} num-bigint = { workspace = true, default-features = true } num-traits = { workspace = true, default-features = true } salsa.workspace = true diff --git a/crates/cairo-lang-syntax/src/node/mod.rs b/crates/cairo-lang-syntax/src/node/mod.rs index 6a821d87ec5..c4ef7d26c43 100644 --- a/crates/cairo-lang-syntax/src/node/mod.rs +++ b/crates/cairo-lang-syntax/src/node/mod.rs @@ -26,6 +26,7 @@ pub mod key_fields; pub mod kind; pub mod stable_ptr; pub mod utils; +pub mod with_db; #[cfg(test)] mod ast_test; diff --git a/crates/cairo-lang-syntax/src/node/with_db.rs b/crates/cairo-lang-syntax/src/node/with_db.rs new file mode 100644 index 00000000000..53735a3a6fc --- /dev/null +++ b/crates/cairo-lang-syntax/src/node/with_db.rs @@ -0,0 +1,49 @@ +use cairo_lang_primitive_token::{PrimitiveSpan, PrimitiveToken, ToPrimitiveTokenStream}; + +use super::SyntaxNode; +use super::db::SyntaxGroup; + +pub struct SyntaxNodeWithDb<'a, Db: SyntaxGroup> { + node: &'a SyntaxNode, + db: &'a Db, +} + +impl<'a, Db: SyntaxGroup> SyntaxNodeWithDb<'a, Db> { + pub fn new(node: &'a SyntaxNode, db: &'a Db) -> Self { + Self { node, db } + } +} + +impl<'a, Db: SyntaxGroup> ToPrimitiveTokenStream for SyntaxNodeWithDb<'a, Db> { + type Iter = SyntaxNodeWithDbIterator<'a, Db>; + + fn to_primitive_token_stream(&self) -> Self::Iter { + // The lifetime of the iterator should extend 'a because it derives from both node and db + SyntaxNodeWithDbIterator::new(Box::new(self.node.tokens(self.db)), self.db) + } +} + +pub struct SyntaxNodeWithDbIterator<'a, Db: SyntaxGroup> { + inner: Box + 'a>, + db: &'a Db, +} + +impl<'a, Db: SyntaxGroup> SyntaxNodeWithDbIterator<'a, Db> { + pub fn new(inner: Box + 'a>, db: &'a Db) -> Self { + Self { inner, db } + } +} + +impl Iterator for SyntaxNodeWithDbIterator<'_, Db> { + type Item = PrimitiveToken; + + fn next(&mut self) -> Option { + self.inner.next().map(|node| { + let span = node.span(self.db).to_str_range(); + PrimitiveToken { + content: node.get_text(self.db), + span: Some(PrimitiveSpan { start: span.start, end: span.end }), + } + }) + } +} diff --git a/scripts/release_crates.sh b/scripts/release_crates.sh index b1e53ea912e..e7d53b6bd6f 100755 --- a/scripts/release_crates.sh +++ b/scripts/release_crates.sh @@ -53,9 +53,9 @@ CRATES_TO_PUBLISH=( ) # Assert that the number of crates to publish is equal to the number of crates in the workspace -# - 4 (the number of crates that are for internal use only). +# - 5 (the number of crates that are for internal use only). NUM_CRATES_IN_WORKSPACE=$(find crates/ -name Cargo.toml | wc -l) -if [ "${#CRATES_TO_PUBLISH[@]}" -ne "$((NUM_CRATES_IN_WORKSPACE - 4))" ]; then +if [ "${#CRATES_TO_PUBLISH[@]}" -ne "$((NUM_CRATES_IN_WORKSPACE - 5))" ]; then echo "The number of crates to publish is not equal to the number of crates in the workspace, new crates were probably added, please update the list of crates to publish." exit 1