Skip to content

Commit

Permalink
Fix broken CI tests and test release code (#894)
Browse files Browse the repository at this point in the history
* Release branch test

* Add v to version number

* add back some missing dependencies

* Add snowflake, cosmos dependencies to buildscript dependency

* Fix snowflake version

* Add snowflake and cosmos as a dependency

* Add few more missing dependencies

* Remove META-INF

* Add sql server dependency

* Add spark-core 3.1 dependency

* add spark sql kafka

* Add jetty

* add compileOnly implementations

* Latest version of cosmos

* Latest version of cosmos

* add sqllite dependency

* upgrade to spark 3.2

* Experiment by changing to only provided

* Change to implementation of snowflake componenets

* update version

* Align all implementations before provided in gradle file

Co-authored-by: rkashyap <[email protected]>
  • Loading branch information
rakeshkashyap123 and rkashyap authored Dec 7, 2022
1 parent e9495d4 commit 77571fd
Show file tree
Hide file tree
Showing 4 changed files with 25 additions and 3 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ build
.bloop/
project/.bloop
metals.sbt
feathr-data-models/src/mainGeneratedDataTemplate/

.bsp/sbt.json

Expand Down
23 changes: 22 additions & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ jar {
from {
configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
}
exclude 'META-INF/*.RSA', 'META-INF/*.SF','META-INF/*.DSA'
zip64 = true
}

dependencies {
Expand All @@ -64,6 +66,16 @@ dependencies {
implementation project(":feathr-impl")
// needed to include data models in jar
extraLibs project(path: ':feathr-data-models', configuration: 'dataTemplate')
implementation 'net.snowflake:snowflake-jdbc:3.13.18'
implementation 'net.snowflake:spark-snowflake_2.12:2.10.0-spark_3.2'
provided 'com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21'
provided 'com.azure.cosmos.spark:azure-cosmos-spark_3-2_2-12:4.11.1'
provided 'com.microsoft.sqlserver:mssql-jdbc:10.2.0.jre8'
provided 'org.eclipse.jetty:jetty-util:9.3.24.v20180605'
provided 'org.apache.kafka:kafka-clients:3.1.0'
provided 'org.apache.spark:spark-core_2.12:3.1.3'
provided 'org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.3'
provided 'org.postgresql:postgresql:42.3.4'
}

ext {
Expand Down Expand Up @@ -105,22 +117,31 @@ project.ext.spec = [
'spark_hive' : "org.apache.spark:spark-hive_$ver.scala_rt:$ver.spark",
'spark_sql' : "org.apache.spark:spark-sql_$ver.scala_rt:$ver.spark",
'spark_catalyst' : "org.apache.spark:spark-catalyst_$ver.scala_rt:$ver.spark",
"spark_sql_kafka" : "org.apache.spark:spark-sql-kafka-0-10_$ver.scala_rt:3.1.3"
],
'scala' : [
'scala_library' : "org.scala-lang:scala-library:$ver.scala",
'scalatest' : "org.scalatest:scalatest_$ver.scala_rt:3.0.0",
],
'avro' : "org.apache.avro:avro:1.10.2",
"avroUtil": "com.linkedin.avroutil1:helper-all:0.2.100",
"azure": "com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21",
'fastutil' : "it.unimi.dsi:fastutil:8.1.1",
'mvel' : "org.mvel:mvel2:2.2.8.Final",
'protobuf' : "com.google.protobuf:protobuf-java:3.19.4",
'protobuf' : "com.google.protobuf:protobuf-java:2.6.1",
'guava' : "com.google.guava:guava:25.0-jre",
'xbean' : "org.apache.xbean:xbean-asm6-shaded:4.10",
'log4j' : "log4j:log4j:1.2.17",
'jetty': "org.eclipse.jetty:jetty-util:9.3.24.v20180605",
'kafka': "org.apache.kafka:kafka-clients:3.1.0",

'json' : "org.json:json:20180130",
'sqlserver': "com.microsoft.sqlserver:mssql-jdbc:10.2.0.jre8",
'postgresql': "org.postgresql:postgresql:42.3.4",
'equalsverifier' : "nl.jqno.equalsverifier:equalsverifier:3.1.12",
'mockito' : "org.mockito:mockito-core:3.1.0",
'snowflake-jdbc' : "net.snowflake:3.13.18",
"spark-snowflake_2.12" : "net.snowflake:2.10.0-spark_3.2",
"mockito_inline": "org.mockito:mockito-inline:2.28.2",
'testing' : "org.testng:testng:6.14.3",
'jdiagnostics' : "org.anarres.jdiagnostics:jdiagnostics:1.0.7",
Expand Down
2 changes: 1 addition & 1 deletion feathr-impl/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,11 @@ dependencies {
implementation spec.product.guava
implementation spec.product.xbean
implementation spec.product.json
implementation spec.product.avroUtil
implementation spec.product.antlr
implementation spec.product.antlrRuntime

implementation spec.product.jackson.jackson_databind
provided spec.product.avroUtil
provided spec.product.typesafe_config
provided spec.product.log4j
provided spec.product.hadoop.common
Expand Down
2 changes: 1 addition & 1 deletion gradle.properties
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
version=0.11.1-rc1
version=v0.10.3-rc5
SONATYPE_AUTOMATIC_RELEASE=true
POM_ARTIFACT_ID=feathr_2.12

0 comments on commit 77571fd

Please sign in to comment.