diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 790ecb5..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,41 +0,0 @@ -version: 2 -jobs: - build: - working_directory: ~/my-project - docker: - - image: circleci/openjdk:8-jdk - environment: - SBT_OPTS: -Xmx3200m - - steps: - - checkout - - restore_cache: - keys: - - lighthouse-{{ checksum "project/Dependencies.scala" }} - - run: - name: Compile and run tests - command: sbt coverage test coverageReport coverageAggregate - - run: - name: Saving test results - command: | - mkdir ~/junit/ - find . -type f -regex ".*/target/test-reports/.*xml" -exec cp {} ~/junit/ \; - when: always - - run: - name: Save coverage results - command: | - mkdir ~/coverage/ - cp -r ./target/scala-2.11/scoverage-report ~/coverage/ - when: always - - store_test_results: # Make test results visible in the UI - path: ~/junit - - store_artifacts: # Make test results downloadable as an artifact - path: ~/junit - - store_artifacts: # Make coverage reports downloadable as an artifact - path: ~/coverage - - save_cache: - key: lighthouse-{{ checksum "project/Dependencies.scala" }} - paths: - - ~/.sbt - - ~/.coursier - - ~/.ivy2 \ No newline at end of file diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 0000000..88939a2 --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,19 @@ +--- +version: 2 +checks: + identical-code: + config: + threshold: 25 + similar-code: + config: + threshold: 60 +plugins: + duplication: + enabled: true + config: + languages: + scala: + scalastyle: + enabled: true + fixme: + enabled: true \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6c18f94 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,59 @@ +name: CI +on: + push: + +jobs: + build: + strategy: + fail-fast: false + matrix: + scala: ["2.12.10", "2.11.12"] + + runs-on: ubuntu-latest + + env: + CC_TEST_REPORTER_ID: ${{ secrets.CODECOV_SECRET }} + SCALA_VERSION: ${{ matrix.scala }} + + steps: + - uses: olafurpg/setup-scala@v7 + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + + - name: Cache Coursier cache + uses: actions/cache@v1.1.2 + with: + path: ~/.coursier + key: ${{ runner.os }}-sbt-coursier-cache-${{ matrix.scala }}-${{ hashFiles(format('{0}/{1}', github.workspace, 'project/Dependencies.scala')) }} + + - name: Cache SBT Ivy2 cache + uses: actions/cache@v1.1.2 + with: + path: ~/.ivy2 + key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles(format('{0}/{1}', github.workspace, 'project/Dependencies.scala')) }}-${{ matrix.scala }} + + - name: Cache SBT + uses: actions/cache@v1.1.2 + with: + path: ~/.sbt + key: ${{ runner.os }}-sbt-sbt-cache-${{ hashFiles(format('{0}/{1}', github.workspace, 'project/Dependencies.scala')) }}-${{ matrix.scala }} + + - name: Install CodeClimate coverage harness + run: | + curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter; + chmod +x ./cc-test-reporter; + ./cc-test-reporter before-build; + + - name: Compile + run: sbt ++${{ matrix.scala }} compile + + - name: Test + id: test + run: sbt ++${{ matrix.scala }} coverage test coverageReport coverageAggregate; + + - name: Send coverage data + run: | + export SCALA_BINARY="${SCALA_VERSION:0:4}"; + ./cc-test-reporter format-coverage -d -t cobertura ./target/scala-$SCALA_BINARY/coverage-report/cobertura.xml && \ + ./cc-test-reporter upload-coverage -d; diff --git a/README.md b/README.md index 11edd64..c4763e9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Lighthouse [![Maven Central](https://maven-badges.herokuapp.com/maven-central/be.dataminded/lighthouse_2.11/badge.svg)](https://maven-badges.herokuapp.com/maven-central/be.dataminded/lighthouse) -[![CircleCI](https://circleci.com/gh/datamindedbe/lighthouse.svg?style=svg)](https://circleci.com/gh/datamindedbe/lighthouse) +[![CI](https://github.com/mrgambal/lighthouse/workflows/CI/badge.svg)](https://github.com/mrgambal/lighthouse/actions?query=workflow%3ACI) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/a0cb9f75da0a4df887b06d37434cfc04)](https://www.codacy.com/app/mLavaert/lighthouse?utm_source=github.com&utm_medium=referral&utm_content=datamindedbe/lighthouse&utm_campaign=Badge_Grade) Lighthouse is a library for data lakes built on top of [Apache Spark](http://spark.apache.org/). diff --git a/build.sbt b/build.sbt index 46f3f41..9d64402 100644 --- a/build.sbt +++ b/build.sbt @@ -6,7 +6,7 @@ import sbt.Opts.resolver.sonatypeStaging lazy val buildSettings = Seq( organization := "be.dataminded", scalaVersion := scala211, - crossScalaVersions := Nil, + crossScalaVersions := supportedScalaVersions, // Ensure code quality scalafmtOnCompile := true, // Memory settings to be able to test with Spark @@ -15,14 +15,14 @@ lazy val buildSettings = Seq( javaOptions ++= Seq( "-Xms768M", "-Xmx2048M", - "-XX:+CMSClassUnloadingEnabled", + "-XX:+UseStringDeduplication", + "-XX:+UseG1GC", "-Dspark.sql.shuffle.partitions=2", "-Dspark.shuffle.sort.bypassMergeThreshold=2", "-Dlighthouse.environment=test" ), scalacOptions ++= Seq( "-deprecation", - "-optimize", "-unchecked", "-Ydelambdafy:inline", "-Ypartial-unification", @@ -61,8 +61,7 @@ lazy val lighthouse = (project in file("lighthouse-core")) lazy val `lighthouse-testing` = (project in file("lighthouse-testing")) .settings( buildSettings, - libraryDependencies ++= Seq(sparkSql, sparkHive, scalaTest, betterFiles), - crossScalaVersions := supportedScalaVersions + libraryDependencies ++= Seq(sparkSql, sparkHive, scalaTest, betterFiles) ) lazy val `lighthouse-demo` = (project in file("lighthouse-demo")) diff --git a/lighthouse-core/src/main/scala/be/dataminded/lighthouse/config/LighthouseConfigurationParser.scala b/lighthouse-core/src/main/scala/be/dataminded/lighthouse/config/LighthouseConfigurationParser.scala index 1341438..e49b8e1 100644 --- a/lighthouse-core/src/main/scala/be/dataminded/lighthouse/config/LighthouseConfigurationParser.scala +++ b/lighthouse-core/src/main/scala/be/dataminded/lighthouse/config/LighthouseConfigurationParser.scala @@ -29,7 +29,7 @@ class LighthouseConfigurationParser extends OptionParser[LighthouseConfiguration opt[String]('e', "environment") .action((environment, config) => config.copy(environment = environment)) - .withFallback(fallbackEnvironment) + .withFallback(() => fallbackEnvironment()) .validate(item => if (item.nonEmpty) success else failure("The configured environment for Lighthouse is empty")) .required() diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 1d78445..57a0a37 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -2,8 +2,8 @@ import sbt._ object Dependencies { - private val amazonSdkVersion = "1.11.659" - private val sparkVersion = "2.4.4" + private val amazonSdkVersion = "1.11.682" + private val sparkVersion = "2.4.5" val sparkCore = "org.apache.spark" %% "spark-core" % sparkVersion % Provided val sparkSql = "org.apache.spark" %% "spark-sql" % sparkVersion % Provided diff --git a/project/build.properties b/project/build.properties index 2930ee9..35b49ae 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.3.7 \ No newline at end of file +sbt.version=1.3.8 \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index 6d5a0f2..e3e5208 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,8 +1,9 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") addSbtPlugin("com.frugalmechanic" % "fm-sbt-s3-resolver" % "0.19.0") -addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.3.0") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.3.1") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.0") +addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.5.0") // Publish to Maven Central addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.8.1")