Merge pull request #966 from doitian/backport-windows-support-to-0.14.0

[ᚬrc/v0.14.0] Backport windows support and sentry cleanup to v0.14.0

.travis.yml

@@ -41,9 +41,12 @@ before_cache:
 deploy:
   provider: releases
   api_key: "$GITHUB_TOKEN"
-  file: "releases/ckb_${TRAVIS_TAG}_${REL_PKG}"
+  file:
+    - "releases/ckb_${TRAVIS_TAG}_${REL_PKG}"
+    - "releases/ckb_${TRAVIS_TAG}_${REL_PKG}.asc"
   skip_cleanup: true
   prerelease: true
+  overwrite: true
   on:
     tags: true
     condition: '"$GITHUB_TOKEN" != "" && "$REL_PKG" != ""'
@@ -102,6 +105,8 @@ matrix:
         - rm -rf $HOME/.cargo/registry
       script:
         - make prod
+        - openssl aes-256-cbc -K $encrypted_82dff4145bbf_key -iv $encrypted_82dff4145bbf_iv -in devtools/ci/travis-secret.asc.enc -out devtools/ci/travis-secret.asc -d
+        - gpg --import devtools/ci/travis-secret.asc
         - devtools/ci/package.sh target/release/ckb
     - name: Package for Linux
       if: 'tag IS present AND env(GITHUB_TOKEN) IS present'
@@ -116,6 +121,8 @@ matrix:
       script:
         - mkdir -p $HOME/.cargo/git
         - docker run --rm -it -w /ckb -v $(pwd):/ckb -v $HOME/.cargo/git:/root/.cargo/git -e OPENSSL_STATIC=1 -e OPENSSL_LIB_DIR=/usr/lib/x86_64-linux-gnu -e OPENSSL_INCLUDE_DIR=/usr/include/openssl $BUILDER_IMAGE make prod
+        - openssl aes-256-cbc -K $encrypted_82dff4145bbf_key -iv $encrypted_82dff4145bbf_iv -in devtools/ci/travis-secret.asc.enc -out devtools/ci/travis-secret.asc -d
+        - gpg --import devtools/ci/travis-secret.asc
         - devtools/ci/package.sh target/release/ckb
     - name: Package for Centos
       if: 'tag IS present AND env(GITHUB_TOKEN) IS present'
@@ -130,4 +137,6 @@ matrix:
       script:
         - mkdir -p $HOME/.cargo/git
         - docker run --rm -it -w /ckb -v $(pwd):/ckb -v $HOME/.cargo/git:/root/.cargo/git $BUILDER_IMAGE scl enable llvm-toolset-7 'make prod'
+        - openssl aes-256-cbc -K $encrypted_82dff4145bbf_key -iv $encrypted_82dff4145bbf_iv -in devtools/ci/travis-secret.asc.enc -out devtools/ci/travis-secret.asc -d
+        - gpg --import devtools/ci/travis-secret.asc
         - devtools/ci/package.sh target/release/ckb

Makefile

@@ -21,6 +21,11 @@ integration: setup-ckb-test ## Run integration tests in "test" dir.
 	cargo build ${VERBOSE}
 	cd test && cargo run ../target/debug/ckb
 
+integration-windows:
+	cp -f Cargo.lock test/Cargo.lock
+	cargo build ${VERBOSE}
+	cd test && cargo run ../target/debug/ckb
+
 integration-release: setup-ckb-test
 	cargo build ${VERBOSE} --release
 	cd test && cargo run --release -- ../target/release/ckb

azure-pipelines.yml

@@ -0,0 +1,86 @@
+trigger:
+  branches:
+    include:
+    - '*'
+  tags:
+    include:
+    - '*'
+
+jobs:
+  - job: UnitTest
+    condition: |
+      and(
+        not(startsWith(variables['Build.SourceBranch'], 'refs/tags/')),
+        or(
+          eq(variables['Build.Reason'], 'PullRequest'),
+          eq(variables['Build.SourceBranch'], 'refs/heads/master'),
+          startsWith(variables['Build.SourceBranch'], 'refs/heads/test')
+        )
+      )
+    pool:
+      vmImage: 'VS2017-Win2016'
+    steps:
+      - template: devtools/azure/windows-dependencies.yml
+        parameters:
+          rustup_toolchain: '1.34.2-x86_64-pc-windows-msvc'
+      - script: make test
+        displayName: Run unit tests
+
+  - job: IntegrationTest
+    condition: |
+      and(
+        not(startsWith(variables['Build.SourceBranch'], 'refs/tags/')),
+        ne(variables['Build.Reason'], 'PullRequest')
+      )
+    pool:
+      vmImage: 'VS2017-Win2016'
+    steps:
+      - template: devtools/azure/windows-dependencies.yml
+        parameters:
+          rustup_toolchain: '1.34.2-x86_64-pc-windows-msvc'
+      - script: make integration-windows
+        displayName: Run integration tests
+
+  - job: Package
+    condition: startsWith(variables['Build.SourceBranch'], 'refs/tags/')
+    pool:
+      vmImage: 'VS2017-Win2016'
+    steps:
+      - template: devtools/azure/windows-dependencies.yml
+        parameters:
+          rustup_toolchain: '1.34.2-x86_64-pc-windows-msvc'
+      - script: make prod
+        displayName: Build
+      - powershell: |
+          mkdir ckb_$(Build.SourceBranchName)_x86_64-pc-windows-msvc
+          cp -r target/release/ckb.exe,README.md,CHANGELOG.md,COPYING,devtools/init,docs ckb_$(Build.SourceBranchName)_x86_64-pc-windows-msvc
+          cp rpc/README.md ckb_$(Build.SourceBranchName)_x86_64-pc-windows-msvc/docs/rpc.md
+        displayName: Prepare archive
+      - task: ArchiveFiles@2
+        inputs:
+          rootFolderOrFile: 'ckb_$(Build.SourceBranchName)_x86_64-pc-windows-msvc'
+          archiveFile: '$(Build.ArtifactStagingDirectory)/ckb_$(Build.SourceBranchName)_x86_64-pc-windows-msvc.zip'
+      - script: choco install -y gpg4win
+        displayName: Install GPG4Win
+      - task: DownloadSecureFile@1
+        inputs:
+          secureFile: azure-secret.asc
+      - script: |
+          set PATH=%PATH%;%ProgramFiles(x86)%/GnuPG/bin
+          gpg --import %DOWNLOADSECUREFILE_SECUREFILEPATH%
+          echo "##vso[task.setvariable variable=PATH;]%PATH%;%ProgramFiles(x86)%/GnuPG/bin"
+        displayName: Setup GPG
+      - script: |
+          gpg -u "Nervos Azure Builder <[email protected]>" -ab "$(Build.ArtifactStagingDirectory)/ckb_$(Build.SourceBranchName)_x86_64-pc-windows-msvc.zip"
+        displayName: Sign Archive
+      - task: GitHubRelease@0
+        inputs:
+          gitHubConnection: nervos-bot
+          repositoryName: nervosnetwork/ckb
+          action: edit
+          tag: $(Build.SourceBranchName)
+          assets: |
+            $(Build.ArtifactStagingDirectory)/*.zip
+            $(Build.ArtifactStagingDirectory)/*.asc
+          assetUploadMode: replace
+          isPreRelease: true

ckb-bin/Cargo.toml

@@ -40,3 +40,4 @@ sentry = "^0.15.4"
 ckb-verification = { path = "../verification" }
 tempfile = "3.0"
 faster-hex = "0.3"
+ckb-script = { path = "../script" }

ckb-bin/src/subcommand/init.rs

@@ -3,6 +3,7 @@ use ckb_resource::{
     TemplateContext, AVAILABLE_SPECS, CKB_CONFIG_FILE_NAME, DEFAULT_SPEC, MINER_CONFIG_FILE_NAME,
     SPEC_DEV_FILE_NAME,
 };
+use ckb_script::Runner;
 
 pub fn init(args: InitArgs) -> Result<(), ExitCode> {
     if args.list_chains {
@@ -12,12 +13,14 @@ pub fn init(args: InitArgs) -> Result<(), ExitCode> {
         return Ok(());
     }
 
+    let runner = Runner::default().to_string();
     let context = TemplateContext {
         spec: &args.chain,
         rpc_port: &args.rpc_port,
         p2p_port: &args.p2p_port,
         log_to_file: args.log_to_file,
         log_to_stdout: args.log_to_stdout,
+        runner: &runner,
     };
 
     let exported = args.locator.exported();

devtools/azure/windows-dependencies.yml

@@ -0,0 +1,17 @@
+parameters:
+  rustup_toolchain: ''
+steps:
+- script: choco install -y llvm
+  displayName: Install LLVM
+- script: choco install -y msys2
+  displayName: Install msys2
+- script: |
+    curl -sSf -o rustup-init.exe https://win.rustup.rs
+    rustup-init.exe -y --default-toolchain ${{ parameters.rustup_toolchain }}
+    set PATH=%PATH%;%USERPROFILE%\.cargo\bin
+    echo "##vso[task.setvariable variable=PATH;]%PATH%;%USERPROFILE%\.cargo\bin"
+  displayName: Install rust
+- script: |
+    rustc --version
+    cargo --version
+  displayName: Test/query rust and cargo versions

devtools/ci/package.sh

@@ -11,6 +11,7 @@ if [ -z "${REL_PKG:-}" ]; then
 fi
 
 PKG_NAME="ckb_${TRAVIS_TAG}_${REL_PKG%%.*}"
+ARCHIVE_NAME="ckb_${TRAVIS_TAG}_${REL_PKG}"
 
 rm -rf releases
 mkdir releases
@@ -28,4 +29,7 @@ if [ "${REL_PKG#*.}" = "tar.gz" ]; then
 else
   zip -r $PKG_NAME.zip $PKG_NAME
 fi
+if [ -n "${GPG_SIGNER:-}" ]; then
+  gpg -u "$GPG_SIGNER" -ab "$ARCHIVE_NAME"
+fi
 popd

docs/get-ckb.md

@@ -5,10 +5,9 @@
 We will publish binaries for each release via [Github Releases]. If your system
 is listed there, you can download the package directory.
 
-[Github Releases]: https://github.com/nervosnetwork/ckb/releases
+CKB releases are signed. It is wise and more secure to check out for their [integrity](integrity-check.md).
 
-There is also a repository [ckb-builds](https://github.com/ckb-builds/ckb-builds/releases) containing the nightly builds from the develop
-branch.
+[Github Releases]: https://github.com/nervosnetwork/ckb/releases
 
 CentOS users please use the `x86_64-unknown-centos-gnu` package, which also
 requires OpenSSL 1.0 to run:
@@ -17,6 +16,13 @@ requires OpenSSL 1.0 to run:
 sudo yum install openssl-libs
 ```
 
+The Windows packages are for experiments only, they have significant
+performance issues, we don't recommend to use them in production environment.
+They requires *The Visual C++ Redistributable Packages*, which can be downloaded
+under section *Other Tools and Frameworks*
+[here](https://visualstudio.microsoft.com/downloads/) or
+[here](https://www.microsoft.com/en-us/download/details.aspx?id=48145).
+
 We also provides docker images, see [how to run CKB with docker](run-ckb-with-docker.md).
 
 ## Build from Source
@@ -63,6 +69,15 @@ scl enable llvm-toolset-7 bash
 
 Remember to run following commands in this console.
 
+#### Windows
+
+Install Visual Studio with Desktop C++ workload, and install following
+packages via [Chocolatey](https://chocolatey.org)
+
+```
+choco install -y llvm msys2
+```
+
 ### Add Environment Variables
 
 If your OS contains pre-compiled `rocksdb` or `snappy` libraries,
@@ -112,3 +127,6 @@ export PATH="$(pwd)/target/release:$PATH"
 # or
 # ln -snf "$(pwd)/target/release/ckb" /usr/local/bin/ckb
 ```
+
+In Windows, use `cargo build --release` instead and the executable is
+`target/release/ckb.exe`.

docs/integrity-check.md

@@ -0,0 +1,54 @@
+# CKB Release Integrity Check
+
+All the binaries available from GitHub releases are signed via following PGP keys.
+
+| Version   | Package              | Unique ID                              | OpenPGP Key                                                                          | Fingerprint                                        |
+| --------- | -------------------- | -------------------------------------- | ------------------------------------------------------------------------------------ | -------------------------------------------------- |
+| >= 0.13.0 | macOS, Linux, CentOS | Nervos Travis Builder <[email protected]> | [F4631C0A](https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x4F37F694F4631C0A) | 64B7 05B5 6078 1FC5 4047  7B82 4F37 F694 F463 1C0A |
+| >= 0.14.0 | Windows              | Nervos Azure Builder <[email protected]>  | [AD748F26](https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x5EBA64ECAD748F26) | 0849 A2D2 4CA7 CFFC FA80  BCD4 5EBA 64EC AD74 8F26 |
+
+You can import the public keys from the keyserver network:
+
+```
+gpg --recv-keys 4F37F694F4631C0A 5EBA64ECAD748F26
+```
+
+Once you have already imported the public keys, please download both the archive and
+the corresponding `.asc` file to verify the signature. For example, to check
+the signature of the file `ckb_v0.13.0_x86_64-apple-darwin.zip`
+
+```
+gpg --verify ckb_v0.13.0_x86_64-apple-darwin.zip.asc ckb_v0.13.0_x86_64-apple-darwin.zip
+```
+
+Note: you should never use a GnuPG version you just downloaded to check the integrity of the source — use an existing, trusted GnuPG installation, e.g., the one provided by your distribution.
+
+If the output of the above command is similar to the following, then either you don't have our public keys or the signature was generated by someone else and the file should be treated suspiciously.
+
+```
+gpg: Signature made Wed 05 Jun 2019 10:12:22 PM UTC using RSA key ID F4631C0A
+gpg: Can't check signature: No public key
+```
+
+If you instead see:
+
+```
+gpg: Signature made Wed 05 Jun 2019 10:12:22 PM UTC using RSA key ID F4631C0A
+gpg: Good signature from "Nervos Travis Builder <[email protected]>"
+gpg: WARNING: This key is not certified with a trusted signature!
+gpg:          There is no indication that the signature belongs to the owner.
+Primary key fingerprint: 64B7 05B5 6078 1FC5 4047  7B82 4F37 F694 F463 1C0A
+```
+
+then you have a copy of our keys and the signatures are valid, but either you have not marked the keys as trusted or the keys are a forgery. In this case, at the very least, you should compare the fingerprints that are shown above.
+
+Ideally, you'll see something like:
+
+```
+gpg: Signature made Wed 05 Jun 2019 10:12:22 PM UTC using RSA key ID F4631C0A
+gpg: checking the trustdb
+gpg: 3 marginal(s) needed, 1 complete(s) needed, PGP trust model
+gpg: depth: 0  valid:   1  signed:   0  trust: 0-, 0q, 0n, 0m, 0f, 1u
+gpg: next trustdb check due at 2023-06-05
+gpg: Good signature from "Nervos Travis Builder <[email protected]>"
+```

miner/src/client.rs

@@ -149,10 +149,6 @@ impl Client {
                 }
                 Err(e) => {
                     error!("rpc call submit_block error: {:?}", e);
-                    sentry::capture_message(
-                        &format!("rpc call submit_block error: {:?}", e),
-                        sentry::Level::Error,
-                    );
                 }
             }
         }