Merge pull request #470 from d3ledger/release/0.3.0

Release/0.3.0

Author: Alexey-N-Chernyshov

.gitignore

@@ -70,6 +70,13 @@ hs_err_pid*
 /btc-dw-bridge/gradle
 /btc-dw-bridge/gradlew
 /btc-dw-bridge/gradlew.bat
+/chain-adapter/build/
+/chain-adapter-integration-test/build/
+/chain-adapter-integration-test/gradle/
+/chain-adapter-integration-test/gradlew
+/bootstrap/build/
+/notary-registration/build/
+/notary-sora-integration-test/build/
 *gradlew.bat
 
 # Ignore Gradle GUI config
@@ -145,14 +152,20 @@ deploy/ethereum/network
 deploy/ethereum/keys/test
 
 # Bitcoin wallets, blockchain and etc.
-deploy/bitcoin/testnet/d3.wallet
 deploy/bitcoin/testnet/blocks/
+deploy/bitcoin/testnet/keys.d3.wallet
+deploy/bitcoin/testnet/transfers.d3.wallet
 deploy/bitcoin/regtest/blocks/
-deploy/bitcoin/regtest/d3.wallet
-deploy/bitcoin/testnet/d3.wallet
-deploy/bitcoin/regtest/d3-test.wallet
 deploy/bitcoin/mainnet/
 
+# Chain-adapter files
+deploy/chain-adapter/
+
+# Bitcoin test wallets
+deploy/bitcoin/regtest/test wallets/
+
 # SMTP configs
 configs/smtp.properties
 
+# Push API configs
+configs/push.properties

Jenkinsfile

@@ -34,11 +34,24 @@ pipeline {
       steps {
         script {
           def scmVars = checkout scm
+          tmp = docker.image("openjdk:8-jdk")
+          env.WORKSPACE = pwd()
+
+          tmp.inside("-e JVM_OPTS='-Xmx3200m' -e TERM='dumb' -v ${env.WORKSPACE}/chain-adapter/build/libs:/home/out") {
+            sh "./gradlew chain-adapter:shadowJar"
+
+          }
+
           DOCKER_NETWORK = "${scmVars.CHANGE_ID}-${scmVars.GIT_COMMIT}-${BUILD_NUMBER}"
           writeFile file: ".env", text: "SUBNET=${DOCKER_NETWORK}"
-          sh "docker-compose -f deploy/docker-compose.yml -f deploy/docker-compose.ci.yml pull"
-          sh(returnStdout: true, script: "docker-compose -f deploy/docker-compose.yml -f deploy/docker-compose.ci.yml up --build -d")
-          sh "docker cp d3-btc-node0-${DOCKER_NETWORK}:/usr/bin/bitcoin-cli deploy/bitcoin/"
+          withCredentials([usernamePassword(credentialsId: 'nexus-d3-docker', usernameVariable: 'login', passwordVariable: 'password')]) {
+            sh "docker login nexus.iroha.tech:19002 -u ${login} -p '${password}'"
+
+            sh "docker-compose -f deploy/docker-compose.yml -f deploy/docker-compose.ci.yml pull"
+            sh(returnStdout: true, script: "docker-compose -f deploy/docker-compose.yml -f deploy/docker-compose.ci.yml up --build -d")
+            sh "docker cp d3-btc-node0-${DOCKER_NETWORK}:/usr/bin/bitcoin-cli deploy/bitcoin/"
+          }
+
           iC = docker.image("openjdk:8-jdk")
           iC.inside("--network='d3-${DOCKER_NETWORK}' -e JVM_OPTS='-Xmx3200m' -e TERM='dumb'") {
             sh "ln -s deploy/bitcoin/bitcoin-cli /usr/bin/bitcoin-cli"
@@ -116,6 +129,7 @@ pipeline {
                 sh "./gradlew eth-withdrawal:shadowJar"
                 sh "./gradlew eth-registration:shadowJar"
                 sh "./gradlew eth-vacuum:shadowJar"
+                sh "./gradlew chain-adapter:shadowJar"
 
                 sh "./gradlew btc-address-generation:shadowJar"
                 sh "./gradlew btc-registration:shadowJar"
@@ -127,13 +141,15 @@ pipeline {
 
               ethRelay = docker.build("nexus.iroha.tech:19002/${login}/eth-relay:${TAG}", "-f docker/eth-relay.dockerfile .")
               ethRegistration = docker.build("nexus.iroha.tech:19002/${login}/eth-registration:${TAG}", "-f docker/eth-registration.dockerfile .")
-              notary = docker.build("nexus.iroha.tech:19002/${login}/notary:${TAG}", "-f docker/notary.dockerfile .")
+              notary = docker.build("nexus.iroha.tech:19002/${login}/notary:${TAG}", "-f docker/eth-deposit.dockerfile .")
               ethWithdrawal = docker.build("nexus.iroha.tech:19002/${login}/eth-withdrawal:${TAG}", "-f docker/eth-withdrawal.dockerfile .")
 
               btcAddressGeneration = docker.build("nexus.iroha.tech:19002/${login}/btc-address-generation:${TAG}", "-f docker/btc-address-generation.dockerfile .")
               btcRegistration = docker.build("nexus.iroha.tech:19002/${login}/btc-registration:${TAG}", "-f docker/btc-registration.dockerfile .")
               btcDwBridge = docker.build("nexus.iroha.tech:19002/${login}/btc-dw-bridge:${TAG}", "-f docker/btc-dw-bridge.dockerfile .")
 
+              chainAdapter = docker.build("nexus.iroha.tech:19002/d3-deploy/chain-adapter:${TAG}", "-f docker/chain-adapter.dockerfile .")
+
               notaryRegistration.push("${TAG}")
 
               ethRelay.push("${TAG}")
@@ -145,6 +161,8 @@ pipeline {
               btcRegistration.push("${TAG}")
               btcDwBridge.push("${TAG}")
 
+              chainAdapter.push("${TAG}")
+
             }
           }
         }

README.md

@@ -16,11 +16,11 @@ Now you can register clients and launch subsystems.
 ## How to run notary application and services in Ethereum main net
 1) Run common services
 2) Provide ethereum passwords `configs/eth/ethereum_password_mainnet.properties` (ask someone from maintainers team about the format)
-3) Deploy Ethereum master contract and relay registry contract, provide notary ethereum accounts `gradle runPreDeployEthereum --args="0x6826d84158e516f631bbf14586a9be7e255b2d23"` 
-4) Run notary service `PROFILE=mainnet gradle runEthNotary`
-5) Run registration service `PROFILE=mainnet gradle runEthRegistration`
-6) Run withdrawal service `PROFILE=mainnet gradle runWithdrawal`
-7) Deploy relay smart contract (one relay per one client registration) `PROFILE=mainnet gradle runDeployRelay`. Ensure relay is deployed on etherscan.io
+3) Deploy Ethereum master contract and relay registry contract, provide notary ethereum accounts `./gradlew runPreDeployEthereum --args="0x6826d84158e516f631bbf14586a9be7e255b2d23"` 
+4) Run deposit service `PROFILE=mainnet ./gradlew runEthDeposit`
+5) Run registration service `PROFILE=mainnet ./gradlew runEthRegistration`
+6) Run withdrawal service `PROFILE=mainnet ./gradlew runWithdrawal`
+7) Deploy relay smart contract (one relay per one client registration) `PROFILE=mainnet ./gradlew runDeployRelay`. Ensure relay is deployed on etherscan.io
 
 Great! So now you can move on and connect frontend application (check back-office repo in d3ledger)
 
@@ -29,7 +29,7 @@ Passwords for Ethereum network may be set in 3 different ways:
 
 1) Using `eth/ethereum_password.properties` file.
 2) Using environment variables(`ETH_CREDENTIALS_PASSWORD`, `ETH_NODE_LOGIN` and `ETH_NODE_PASSWORD`).
-3) Using command line arguments. For example `gradle runEthNotary -PcredentialsPassword=test -PnodeLogin=login -PnodePassword=password`
+3) Using command line arguments. For example `./gradlew runEthDeposit -PcredentialsPassword=test -PnodeLogin=login -PnodePassword=password`
 
 Configurations have the following priority:
 
@@ -38,21 +38,34 @@ Command line args > Environment variables > Properties file
 ## How to run notary application and services in Bitcoin main net
 1) Run common services
 2) Create `.wallet` file (ask maintainers how to do that) and put it to desired location
-3) Run address generation process using `PROFILE=mainnet gradlew runBtcAddressGeneration`
-4) Create change address by running `gradlew generateBtcChangeAddress`
-5) Create few free addresses(addresses that may be registered by clients lately) `gradlew generateBtcFreeAddress`
-6) Run registration service `PROFILE=mainnet gradle runBtcRegistration`
-7) Run notary service `PROFILE=mainnet gradle runBtcDepositWithdrawal`
-
+3) Run address generation process using `PROFILE=mainnet ./gradlew runBtcAddressGeneration`
+4) Create change address by running `./gradlew generateBtcChangeAddress`
+5) Create few free addresses(addresses that may be registered by clients lately) `./gradlew generateBtcFreeAddress`
+6) Run registration service `PROFILE=mainnet ./gradlew runBtcRegistration`
+7) Run notary service `PROFILE=mainnet ./gradlew runBtcDepositWithdrawal`
+
+## How to run notification services
+1) Create SMTP configuration file located at `configs/smtp.properties`(see test example `configs/smtp_test.properties`). This file contains SMTP server credentials.
+2) Create Push API configuration file located at `configs/push.properties`(see test example `configs/push_test.properties`). This file contains VAPID keys. You can generate keys by yourself using [webpush-libs tutorial](https://github.com/web-push-libs/webpush-java/wiki/VAPID).
+3) Run gralde command `./gradlew runNotifications`
+
+## AWS SES Integration
+SMTP configuration file `configs/smtp.properties` must be modified in order to be able to send email notifications via AWS SES. 
+```
+smtp.host=email-smtp.eu-west-1.amazonaws.com
+smtp.port=25
+smtp.userName=ask maintainers
+smtp.password=ask maintainers
+```
 ## Testing
-`gradle test` for unit tests
+`./gradlew test` for unit tests
 
-`gradle integrationTest` for integation tests
+`./gradlew integrationTest` for integation tests
 
 ## Testing Bitcoin
-`gradlew btcSendToAddress -Paddress=<address> -PamountBtc=<amount>` — sends `<amount>` BTC to `<address>` in Bitcoin regtest network
+`./gradlew btcSendToAddress -Paddress=<address> -PamountBtc=<amount>` — sends `<amount>` BTC to `<address>` in Bitcoin regtest network
 
-`gradlew btcGenerateBlocks -Pblocks=<blocks>` — generates `<blocks>` in Bitcoin regtest network
+`./gradlew btcGenerateBlocks -Pblocks=<blocks>` — generates `<blocks>` in Bitcoin regtest network
 
 ## Troubleshooting
 
@@ -65,4 +78,7 @@ Command line args > Environment variables > Properties file
 127.0.0.1 d3-notary
 127.0.0.1 d3-eth-node0
 127.0.0.1 d3-btc-node0
+127.0.0.1 d3-rmq
+127.0.0.1 d3-brvs
+127.0.0.1 d3-brvs-mongodb
 ```

bootstrap/ReadMe.md

@@ -0,0 +1,33 @@
+# Bootstrap service
+
+## API documentation
+* SwaggerUI - 'http://localhost:8080/swagger-ui.html' In SwaggerUi it is also possible to make test calls through browser
+* Json representation - 'http://localhost:8080/v2/api-docs' In Firefox json is more readable, so try to use it.
+
+
+## Description
+Service provides possibilities to automate DevOps activities.
+
+It is usable for test environments and production. Only difference is that for test environments we 
+may generate blockchain credentials using this service but for production owners of private keys 
+should provide public keys for us.
+
+Functionality:
+* Creation of Iroha credentials (private and public keys)
+* Creation of Iroha Genesis Block 
+
+## Startup Configuration
+You should provide environment parameter `btc.network` with one of three values: `TestNet3, MainNet, RegTest`
+Else `btc.network` parameter will be taken from application.properties file
+
+## How to develop
+To Add generation of genesis block for new project and/or environment Create a class which should 
+implement `GenesisInterface`
+
+* `fun getProject(): String` - should provide name of project
+* `fun getEnvironment(): String` - should provide name of environment eg 'local_my', 'test', 'prod'
+* `fun createGenesisBlock(accounts:List<IrohaAccountDto>, peers:List<Peer>, blockVersion:String = "1"): String`
+This function should return string json representatin of genesis block. It takes Iroha peers and accounts as arguments from API request.
+* `fun getAccountsNeeded(): List<AccountPrototype>` - This function should return Accounts which have to present in genesis block to generate credentials for.
+
+Example realisation for D3: `genesis.d3.D3TestGenesisFactory`

bootstrap/build.gradle

@@ -0,0 +1,88 @@
+group 'd3'
+version '1.0'
+
+buildscript {
+    repositories {
+        mavenCentral()
+        jcenter()
+    }
+    dependencies {
+        //Spring-boot
+        classpath "org.springframework.boot:spring-boot-gradle-plugin:$spring_boot_version"
+    }
+}
+
+apply plugin: 'application'
+apply plugin: "kotlin-spring" // See https://kotlinlang.org/docs/reference/compiler-plugins.html#kotlin-spring-compiler-plugin
+apply plugin: 'com.github.johnrengelman.shadow'
+shadowDistZip.classifier = 'shadow'
+shadowDistTar.classifier = 'shadow'
+
+dependencies {
+
+    final def springfoxSwagger2Version = '2.9.2'
+    final def springBootVersion = '2.1.3.RELEASE'
+    final def kotlinVersion = '1.3.10'
+
+    implementation project(":notary-commons")
+    implementation project(":eth")
+
+    // kotlin related
+    compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion"
+    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.0.1'
+    compile 'io.github.microutils:kotlin-logging:1.6.24'
+    testCompile "org.jetbrains.kotlin:kotlin-test:$kotlinVersion"
+
+    //Iroha-java related
+    implementation('com.github.hyperledger.iroha-java:testcontainers:3.2.1') {
+        exclude group: 'org.slf4j'
+    }
+
+    //Spring-boot
+    implementation("org.springframework.boot:spring-boot-starter:$springBootVersion") {
+        exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging'
+    }
+    implementation("org.springframework.boot:spring-boot-starter-web:$springBootVersion") {
+        exclude group: 'org.springframework.boot', module: 'spring-boot-starter-tomcat'
+        exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging'
+        exclude group: 'org.hibernate'
+    }
+    implementation "io.springfox:springfox-swagger2:$springfoxSwagger2Version"
+    implementation "io.springfox:springfox-swagger-ui:$springfoxSwagger2Version"
+
+    //TODO this is very dirty hack. This dependency must be removed after migration to newer version of Spring Boot
+    // https://mvnrepository.com/artifact/org.hibernate/hibernate-validator
+
+    implementation 'org.hibernate:hibernate-validator:6.0.14.Final'
+
+    implementation "org.springframework.boot:spring-boot-starter-jetty:$springBootVersion"
+
+    implementation "commons-logging:commons-logging:1.2"
+
+    compile 'com.google.protobuf:protobuf-java-util:3.6.1'
+    compile 'org.bitcoinj:bitcoinj-core:0.14.7'
+
+    //Web3j
+    compile 'org.web3j:crypto:4.1.1'
+    compile 'org.web3j:core:4.1.1'
+
+    testCompile "org.springframework.boot:spring-boot-starter-test:$springBootVersion"
+
+}
+
+sourceCompatibility = 1.8
+
+
+mainClassName = "jp.co.soramitsu.bootstrap.BootstrapMain"
+
+task runBootstrap(type: JavaExec ) {
+    main = mainClassName
+    classpath = sourceSets.main.runtimeClasspath
+    setWorkingDir("$rootDir/")
+}
+
+sonarqube {
+    properties {
+        property "sonar.projectKey", "bootstrap"
+    }
+}

bootstrap/src/main/kotlin/jp/co/soramitsu/bootstrap/Application.kt

@@ -0,0 +1,22 @@
+
+@file:JvmName("BootstrapMain")
+
+package jp.co.soramitsu.bootstrap
+
+import mu.KLogging
+import org.springframework.boot.SpringApplication
+import org.springframework.boot.autoconfigure.SpringBootApplication
+import org.springframework.context.annotation.ComponentScan
+
+@SpringBootApplication
+@ComponentScan(basePackages = ["jp.co.soramitsu.bootstrap"])
+class Application
+
+    private val logger = KLogging().logger
+
+    fun main(args: Array<String>) {
+        val app = SpringApplication(Application::class.java)
+        app.run(*args)
+    }
+
+

bootstrap/src/main/kotlin/jp/co/soramitsu/bootstrap/config/BootstrapConfig.kt

@@ -0,0 +1,18 @@
+package jp.co.soramitsu.bootstrap.config
+
+import jp.co.soramitsu.bootstrap.genesis.d3.D3TestGenesisFactory
+import jp.co.soramitsu.bootstrap.genesis.GenesisInterface
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import springfox.documentation.swagger2.annotations.EnableSwagger2
+
+@Configuration
+@EnableSwagger2
+class BootstrapConfig {
+
+    @Bean
+    fun genesisFactories(): List<GenesisInterface> {
+        return listOf(D3TestGenesisFactory())
+    }
+}
+

bootstrap/src/main/kotlin/jp/co/soramitsu/bootstrap/controller/BtcController.kt

@@ -0,0 +1,47 @@
+package jp.co.soramitsu.bootstrap.controller
+
+import jp.co.soramitsu.bootstrap.dto.BtcNetwork
+import jp.co.soramitsu.bootstrap.dto.BtcWallet
+import mu.KLogging
+import org.bitcoinj.core.NetworkParameters
+import org.bitcoinj.wallet.Wallet
+import org.springframework.beans.factory.annotation.Value
+import org.springframework.http.HttpStatus
+import org.springframework.http.ResponseEntity
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RestController
+import java.io.ByteArrayOutputStream
+import javax.xml.bind.DatatypeConverter
+
+@RestController
+@RequestMapping("/btc")
+class BtcController {
+
+    private val log = KLogging().logger
+    @Value("\${btc.network}")
+    private lateinit var network: BtcNetwork
+
+    @GetMapping("/create/wallet")
+    fun createWallet(): ResponseEntity<BtcWallet> {
+        try {
+            return ResponseEntity.ok<BtcWallet>(createWallet(network.params))
+        } catch (e: Exception) {
+            log.error("Error creating Bitcoin wallet", e)
+            val response = BtcWallet(network = network)
+            response.errorCode = e.javaClass.simpleName
+            response.message = e.message
+            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(response)
+        }
+    }
+
+    private fun createWallet(netParams: NetworkParameters): BtcWallet {
+        val wallet = Wallet(netParams)
+
+        val out = ByteArrayOutputStream(1024)
+        wallet.saveToFileStream(out)
+        val walletBytes = out.toByteArray()
+        out.reset()
+        return BtcWallet(DatatypeConverter.printBase64Binary(walletBytes), network)
+    }
+}