From 38ce2d16bf129649597b3233887ada6116d0b1c3 Mon Sep 17 00:00:00 2001 From: zhiruiwang Date: Tue, 31 Jul 2018 10:37:50 +0800 Subject: [PATCH] add doc Signed-off-by: zhiruiwang --- .DS_Store | Bin 0 -> 6148 bytes ...with_diskquota\347\277\273\350\257\221.md" | 79 ++++++ ...with_dragonfly\347\277\273\350\257\221.md" | 60 ++++ ...50\351\207\217\345\256\236\350\267\265.md" | 268 ++++++++++++++++++ 4 files changed, 407 insertions(+) create mode 100644 .DS_Store create mode 100644 "blog-cn/pouch_with_diskquota\347\277\273\350\257\221.md" create mode 100644 "blog-cn/pouch_with_dragonfly\347\277\273\350\257\221.md" create mode 100644 "blog-en/PouchContainer \345\267\245\347\250\213\350\264\250\351\207\217\345\256\236\350\267\265.md" diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..baa8f4a5e2356f770bd386b9764f6a70ab4539ee GIT binary patch literal 6148 zcmeHK!A=`75FJAxn^uS%Dp78}RO%rh5-voj(l!kT4wX<-wFjtRmyKYhWgR7(Mv$i3 z?|?7hSNa3}obo(&P)N!VM5#v_zp-b=mY>$1H4%wMKdKUyiO7R$EWAMT7twyLGcspK zia^10tkE&0yNNmulbL8Y;23ZWJT(UN*)38?2`M_EiSw&Mc=WZf4SoBE1=dV-IOui4 zgrk3joh0sM8ymmH+LK<>~o5<5`Cf_dQVOI2wbN#icGB-^CniNcth98gE3&ge(&Xjt>7kh3^)e< zRR;9^AfXz4i@8C*I?(VF09e2<3vB(6Uf<-?{(&e>chX90QJlXT^XhG+WIE zwq$GT%;t2hmCz%o79E!x= 2.6| \>= 4.5 +xfs | \>= 2.6 | \>= 3.10 + +尽管每个文件系统在相关内核版本下支持diskquota，但是用户仍然需要安装[quota-tools-4.04](https://nchc.dl.sourceforge.net/project/linuxquota/quota-tools/4.04/quota-4.04.tar.gz)。 +这个配额工具还没有打包在PouchContainer软件包管理器中。我们会在未来将其完成。 + +## 开始 + +PouchContainer为容器涉及底层文件系统提供了两种方式。一个是容器根文件系统，另一个是从主机（容器外）到内部的容器卷标绑定。两个维度都包含在PouchContainer。 + +### 容器根文件系统diskquota + +用户可以为一个以创建的容器根文件系统，设置标志`--disk-quota`来限制磁盘空间的使用，例如`--disk-quota 10g`。成功设置这个标志后，我们可以通过命令`df -h`得到根文件系统的大小是10G。 同时，这说明diskquota生效了。 + +``` +$ pouch run -ti --disk-quota 10g registry.hub.docker.com/library/busybox:latest df -h +Filesystem Size Used Available Use% Mounted on +overlay 10.0G 24.0K 10.0G 0% / +tmpfs 64.0M 0 64.0M 0% /dev +shm 64.0M 0 64.0M 0% /dev/shm +tmpfs 64.0M 0 64.0M 0% /run +tmpfs 64.0M 0 64.0M 0% /proc/kcore +tmpfs 64.0M 0 64.0M 0% /proc/timer_list +tmpfs 64.0M 0 64.0M 0% /proc/sched_debug +tmpfs 1.9G 0 1.9G 0% /sys/firmware +tmpfs 1.9G 0 1.9G 0% /proc/scsi +``` + +### 卷标diskquota + +用户也可以在创建时设置卷标的磁盘限制。通过增加一个`--option` 或 `-o` 标志，指定磁盘空间限制为一个期望的数据变得十分容易，例如 `-o size=10g`。 + +创建限制卷标的diskquota后，用户可以将这个卷标绑定到一个正在执行的容器。在下面的例子中，执行命令 +`pouch run -ti -v volume-quota-test:/mnt registry.hub.docker.com/library/busybox:latest df -h`。 +在运行着的容器中，目录`/mnt` 被限制到10GB的大小。 + +``` +$ pouch volume create -n volume-quota-test -d local -o mount=/data/volume -o size=10g +Name: volume-quota-test +Scope: +Status: map[mount:/data/volume sifter:Default size:10g] +CreatedAt: 2018-3-24 13:35:08 +Driver: local +Labels: map[] +Mountpoint: /data/volume/volume-quota-test +$ pouch run -ti -v volume-quota-test:/mnt registry.hub.docker.com/library/busybox:latest df -h +Filesystem Size Used Available Use% Mounted on +overlay 20.9G 212.9M 19.6G 1% / +tmpfs 64.0M 0 64.0M 0% /dev +shm 64.0M 0 64.0M 0% /dev/shm +tmpfs 64.0M 0 64.0M 0% /run +/dev/sdb2 10.0G 4.0K 10.0G 0% /mnt +tmpfs 64.0M 0 64.0M 0% /proc/kcore +tmpfs 64.0M 0 64.0M 0% /proc/timer_list +tmpfs 64.0M 0 64.0M 0% /proc/sched_debug +tmpfs 1.9G 0 1.9G 0% /sys/firmware +tmpfs 1.9G 0 1.9G 0% /proc/scsi +``` diff --git "a/blog-cn/pouch_with_dragonfly\347\277\273\350\257\221.md" "b/blog-cn/pouch_with_dragonfly\347\277\273\350\257\221.md" new file mode 100644 index 0000000..5c4d429 --- /dev/null +++ "b/blog-cn/pouch_with_dragonfly\347\277\273\350\257\221.md" @@ -0,0 +1,60 @@ +# 具有Dragonfly的PouchContainer + +容器技术有利于促进IT操作和维护，但同时为图像的分布带来了巨大的挑战。图像可以像几个Gibs一样大，拉动图像会非常慢，更不用说多个拉动请求或低网络带宽的时候。Dragonfly在这里可以发挥重要作用，作为一种P2P技术，它提供了非常高效的分布，避免了图像分发成为容器技术的瓶颈。 + +## 什么是Dragonfly + +[Dragonfly](https://github.com/alibaba/Dragonfly#installation) 是一个P2P文件分布系统。它解决了大规模文件分发场景中耗时分布，低成功率和浪费带宽的问题。 Dragonfly显着提高了数据预热和大规模容器图像分发等服务功能。 + +### 组成 + +#### 服务器 + +服务器由集群管理器组成，它从源代码下载文件并构建P2P网络。 + +#### 客户端 + +客户端包含了两部分：dfget和proxy。Dfget是一种用于下载文件的主机端工具，proxy用于拦截http请求并将它们发送到dfget。 + +## 安装dragonfly + +1.安装服务器 + +服务器可以两种方式部署，在物理机器上或在容器上，[install dragonfly server](https://github.com/alibaba/Dragonfly/blob/master/docs/install_server.md)的步骤很简单。 + +2.安装客户端 + +- 下载[client package](https://github.com/alibaba/Dragonfly/blob/master/package/df-client.linux-amd64.tar.gz). +- 解压缩包`tar xzvf df-client.linux-amd64.tar.gz -C /usr/local/bin`，或者你可以解压到任何你喜欢的目录，但是记得将客户端路径添加到PATH环境中， `export PATH=$PATH:/usr/local/bin/df-client`。 +- 添加服务端ip到客户端配置文件中， `/etc/dragonfly.conf`，nodeIp是部署服务器主机的IP。 + +``` +[node] +address=nodeIp1,nodeIp2,... +``` + +关于详细的安装信息，可以从 [install dragonfly client](https://github.com/alibaba/Dragonfly/blob/master/docs/install_client.md)中获得。 + +## 运行具有dragonfly的PouchContainer + +1.开启dragonfly代理 + +开启dragonfly代理，`/usr/local/bin/df-client/df-daemon --registry https://reg.docker.alibaba-inc.com`，你可以增加标志 `--verbose` 来获取debug日志。 Dragonfly日志可在文件`~/.small-dragonfly/logs`中找到。 + +更多的dragonfly用法信息可在[dragonfly usage](https://github.com/alibaba/Dragonfly/blob/master/docs/usage.md)中找到。 + +2.添加下面的配置到PouchContain配置文件`/etc/pouch/config.json`中 + +``` +{ + "image-proxy": "http://127.0.0.1:65001" +} +``` + +3.拉取一个图像`reg.docker.alibaba-inc.com/base/busybox:latest`，你可在`~/.small-dragonfly/logs/dfdaemon.log`中找到下列输出，这意味着dragonfly正在工作。 + +``` +time="2018-03-06 20:08:00" level=debug msg="pre access:http://storage.docker.aliyun-inc.com/docker/registry/v2/blobs/sha256/1b/1b5110ff48b0aa7112544e1666cc7199f812243ded4128f0a1b2be027c7 38bec/data?Expires=1520339335&OSSAccessKeyId=LTAIfYaNrksx0ktL&Signature=fVEYIQzIaXyqIcAhypbmzaUx5x8%3D" +time="2018-03-06 20:08:00" level=debug msg="post access:http://storage.docker.aliyun-inc.com" +time="2018-03-06 20:08:00" level=debug msg="pre access:http://storage.docker.aliyun-inc.com/docker/registry/v2/blobs/sha256/98/9869810a78644038c8d37a5a3344de0217cb37bcc2caa2313036c6948b0 ee8da/data?Expires=1520339335&OSSAccessKeyId=LTAIfYaNrksx0ktL&Signature=Tx7JYU07Gap8RfasvCe0JGAUCo4%3D" +``` diff --git "a/blog-en/PouchContainer \345\267\245\347\250\213\350\264\250\351\207\217\345\256\236\350\267\265.md" "b/blog-en/PouchContainer \345\267\245\347\250\213\350\264\250\351\207\217\345\256\236\350\267\265.md" new file mode 100644 index 0000000..92579c2 --- /dev/null +++ "b/blog-en/PouchContainer \345\267\245\347\250\213\350\264\250\351\207\217\345\256\236\350\267\265.md" @@ -0,0 +1,268 @@ +# 0.Preface + +As the iteration and perfection of the functionality of [PouchContainer](https://github.com/alibaba/pouch) the project has become huge gradually, attracting a number of external developers to participate in the development of the project. The coding habits of each contributor are different, the responsibility of code reviewer is not only to focus on logical correctness and performance issues, but also on code style, it is because that a unified code specification is a prerequisite for maintaining project code maintainability. In addition to the unified project code style, the coverage and stability of test cases is also the focus of the project. We can assume that how to ensure that code update does not affect existing features in the absence of regression test cases everytime ? + +This article will share the specifications of PouchContainer's in code style and the practices of golang in test unit. + +# 1.The Unified specification of coding style + +PouchContainer is a project built by the golang language, which uses shell scripts to perform automated operations such as compiling and packaging. In addition to golang and shell scripts, PouchContainer also contains a large number of Markdown-style documents, which is the entry point for users to understand and understand PouchContainer. Its standard layout and correct spelling are also the focus of the project. The following sections describe the tools and usage scenarios that PouchContainer uses in coding style specifications. + +## 1.1 Golinter - The Unified coding format + +The grammar design of Golang is simple, and the community has a complete [CodeReview](https://github.com/golang/go/wiki/CodeReviewComments) guide from the beginning, so most of the golang projects have the same code style, and rarely fall into the unnecessary **religious** dispute. On the basis of the community, PouchContainer also defines some specific rules to stipulate the developer to ensure the readability of the code, the specific content can be read [here](https://github.com/alibaba/pouch/blob/master/docs/contributions/code_styles.md#additional-style-rules). + +However, it is difficult to ensure that the project code style is consistent by relying on written agreements to make specifications. So golang, like other languages, provides the official toolchain, such as [golint](https://github.com/golang/lint), [gofmt](https://golang.org/cmd/gofmt/), [goimports](https://github.com/golang/tools/blob/master/cmd/goimports/doc.go), and [go vet](https://golang.org/cmd/vet/), which can be used to check and unify code styles before compilation, and provide the automated possibility to subsequent processes, such as code review. Currently, PouchContainer runs the above code checking tool in CircleCI in **every** Pull Request submitted by the developer. If the inspection tool displays an exception, the code reviewer has the right to **refuse** the review and may even reject the merge code. + +In addition to the tools provided by official, we can also select third-party code inspection tools in the open source community, such as [errcheck](https://github.com/kisielk/errcheck) to check if the developer has processed the error returned by the function. However, these tools do not have a uniform output format, which makes it difficult to integrate the output of different tools. Fortunately, some people in the open source community have implemented this unified interface, [gometalinter](https://github.com/alecthomas/gometalinter), which can integrate various code checking tools. The recommended combination is: + +* [golint](https://github.com/golang/lint) - Google's (mostly stylistic) linter. +* [gofmt -s](https://golang.org/cmd/gofmt/) - Checks if the code is properly formatted and could not be further simplified. +* [goimports](https://godoc.org/golang.org/x/tools/cmd/goimports) - Checks missing or unreferenced package imports. +* [go vet](https://golang.org/cmd/vet/) - Reports potential errors that otherwise compile. +* [varcheck](https://github.com/opennota/check) - Find unused global variables and constants. +* [structcheck](https://github.com/opennota/check) - Find unused struct fields +* [errcheck](https://github.com/kisielk/errcheck) - Check that error return values are used. +* [misspell](https://github.com/client9/misspell) - Finds commonly misspelled English words. + +Each project can be customized according to your needs gometalinter package. + +## 1.2 Shellcheck - Reduce potential problems with shell scripts + +Although shell scripts are powerful, they still require syntax checking to avoid potential, unpredictable errors. For example, the definition of unused variables, although it does not affect the use of the script, but its existence will become a burden on the project maintainer. + +```bash +#!/usr/bin/env bash + +pouch_version=0.5.x + +dosomething() { + echo "do something" +} + +dosomething +``` + +PouchContainer will use [shellcheck](https://github.com/koalaman/shellcheck) to check the shell script in the current project. Taking the above code as an example, shellcheck detection will get a warning of unused variables. This tool can detect potential problems with shell scripts during the code review phase, reducing the chance of runtime errors. + +```bash +In test.sh line 3: +pouch_version=0.5.x +^-- SC2034: pouch_version appears unused. Verify it or export it. +``` + +The current continuous integration task of PouchContainer scans scans the `.sh` scripts in the project and checks them one by one using shellcheck. See [here](https://github.com/koalaman/shellcheck/wiki) for details. + +> NOTE: When the shellcheck check is too strict, the project can be bypassed by a comment, or a check can be closed in the project. Specific inspection rules can be found [here](https://github.com/koalaman/shellcheck/wiki). + +## 1.3 Markdownlint - Unified Document Formatting + +PouchContainer is an open source project whose documentation is as important as the code, because documentation is the best way for users to understand PouchContainer. The document is written in the form of markdown, and its formatting and spelling errors are the focus of the project. + +As with the code, there may be a missed judgment by only basing on text convention, so PouchContainer uses [markdownlint](https://github.com/markdownlint/markdownlint) and [misspell](https://github.com/client9/misspell) to check the document format and spelling errors. These checks have the same status as `golint` and will run in CircleCI every time Pull Request occurs. Once an exception occurs, code reviewers have the right to **refuse** to review or merge the code. + +PThe current continuous integration task of PouchContainer checks the the format of markdown document in the project and also checks the spelling in all files. The configuration can be found [here](https://github.com/alibaba/pouch/blob/master/.circleci/config.yml#L13-L20). + +> NOTE: When the markdownlint requirement is too strict, the corresponding check can be closed in the project. Specific inspection items can be found [here](https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md). + +## 1.4 summary + +All of the above are style discipline issues, and PouchContainer automates code specification detection and integrates into each code review to help reviewers identify potential problems. + +# 2.How to write a unit test for golang + +Unit testing can be used to ensure the correctness of a single module. In the pyramid of test areas, the wider the unit test coverage and the more comprehensive coverage, the more it can reduce the debugging costs of integration testing and end-to-end testing. In a complex system, the longer the link processed by the task, the higher the cost of the location problem, especially the problems caused by small modules. The following sections share a summary of PouchContainer's preparation of golang unit test cases. + +## 2.1 Table-Driven Test - DRY + +A simple understanding of unit testing is to give a given input to a function to determine if the expected output can be obtained. When the function being tested has a variety of input scenarios, we can organize our test cases in the form of Table-Driven, as shown in the next code. Table-Driven uses arrays to organize test cases and validate the correctness of the function by loop execution. + +```bash +// from https://golang.org/doc/code.html#Testing +package stringutil + +import "testing" + +func TestReverse(t *testing.T) { + cases := []struct { + in, want string + }{ + {"Hello, world", "dlrow ,olleH"}, + {"Hello, 世界", "界世 ,olleH"}, + {"", ""}, + } + for _, c := range cases { + got := Reverse(c.in) + if got != c.want { + t.Errorf("Reverse(%q) == %q, want %q", c.in, got, c.want) + } + } +} +``` + +To facilitate debugging and maintenance of test cases, we can add some auxiliary information to describe the current test. For example, if [reference](https://github.com/alibaba/pouch/blob/master/pkg/reference/parse_test.go#L54) wants to test the input of [punycode](https://en.wikipedia.org/wiki/Punycode), if you don't include the word `punycode`, they may not know the difference between `xn--bcher-kva.tld/redis:3` and `docker.io/library/redis:3`. + +```bash +{ + name: "Normal", + input: "docker.io/library/nginx:alpine", + expected: taggedReference{ + Named: namedReference{"docker.io/library/nginx"}, + tag: "alpine", + }, + err: nil, +}, { + name: "Punycode", + input: "xn--bcher-kva.tld/redis:3", + expected: taggedReference{ + Named: namedReference{"xn--bcher-kva.tld/redis"}, + tag: "3", + }, + err: nil, +} +``` + +However, some functions are more complicated, so that one input cannot be used as a complete test case. Such as [TestTeeReader](https://github.com/golang/go/blob/release-branch.go1.9/src/io/io_test.go#L284),TeeReader read the `hello, world` from the buffer, and the data has been read. If you read it again, the expected behavior is an end-of-file error. Such a test case needs to be done in a single case, without the need to fit intentionally the form of Table-Driven. + +To put it simply, if you test a function that needs to copy most of the code, theoretically the test code can be extracted and used to organize test cases using Table-Driven. **Don`t Repeat Yourself** is our principle. + +> NOTE: The Table-Driven organization is recommended by the golang community. Please check [here](https://github.com/golang/go/wiki/TableDrivenTests) for details. + +## 2.2 Mock - Simulating external dependencies + +The problems with dependencies often occur during the testing process. For example, the PouchContainer client requires an HTTP server, but this is too heavy for the unit, and this is a category of integration testing. So how do you complete this part of the unit test? + +In the world of golang, the implementation of the interface belongs to [Duck Type](https://en.wikipedia.org/wiki/Duck_typing). An interface can have a variety of implementations, as long as the implementation conforms to the interface definition. If the external dependencies are constrained by the interface, then the dependency behavior is simulated in the unit test. The following content will share two common test scenarios. + +### 2.2.1 RoundTripper + +Take the PouchContainer client test as an example. The PouchContainer client uses [http.Client](https://golang.org/pkg/net/http/#Client). And the http.Client uses the [RoundTripper](https://golang.org/pkg/net/http/#RoundTripper) interface to perform an HTTP request, which allows developers to customize the logic for sending HTTP requests. This is also an important reason why golang can perfectly support the HTTP 2 protocol on an original basis. + +```bash +http.Client -> http.RoundTripper [http.DefaultTransport] +``` + +For the PouchContainer client, the test focus is mainly on whether the incoming destination address is correct, whether the incoming query is reasonable, and whether the result can be returned normally. So before testing, developers need to prepare the corresponding RoundTripper implementation, which is not responsible for the actual business logic, it is only used to determine whether the input meets expectations. + +As shown in the next code, PouchContainer `newMockClient` accepts custom request processing logic. In the test case of using to remove the image, the developer determines in the custom logic whether the destination address and the HTTP Method are DELETE, so that the functional test can be completed without starting the HTTP Server. + +```bash +// https://github.com/alibaba/pouch/blob/master/client/client_mock_test.go#L12-L22 +type transportFunc func(*http.Request) (*http.Response, error) + +func (transFunc transportFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return transFunc(req) +} + +func newMockClient(handler func(*http.Request) (*http.Response, error)) *http.Client { + return &http.Client{ + Transport: transportFunc(handler), + } +} + +// https://github.com/alibaba/pouch/blob/master/client/image_remove_test.go +func TestImageRemove(t *testing.T) { + expectedURL := "/images/image_id" + + httpClient := newMockClient(func(req *http.Request) (*http.Response, error) { + if !strings.HasPrefix(req.URL.Path, expectedURL) { + return nil, fmt.Errorf("expected URL '%s', got '%s'", expectedURL, req.URL) + } + if req.Method != "DELETE" { + return nil, fmt.Errorf("expected DELETE method, got %s", req.Method) + } + + return &http.Response{ + StatusCode: http.StatusNoContent, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + }, nil + }) + + client := &APIClient{ + HTTPCli: httpClient, + } + + err := client.ImageRemove(context.Background(), "image_id", false) + if err != nil { + t.Fatal(err) + } +} +``` + +### 2.2.2 MockImageManager + +For dependencies between internal packages, such as the PouchContainer Image API Bridge depends on the PouchContainer Daemon ImageManager, and the dependency behavior is defined by interface. If we want to test the logic of Image Bridge, we don't have to start containerd , we just need to implement the corresponding Daemon ImageManager like RoundTripper. + +```bash +// https://github.com/alibaba/pouch/blob/master/apis/server/image_bridge_test.go +type mockImgePull struct { + mgr.ImageMgr + handler func(ctx context.Context, imageRef string, authConfig *types.AuthConfig, out io.Writer) error +} + +func (m *mockImgePull) PullImage(ctx context.Context, imageRef string, authConfig *types.AuthConfig, out io.Writer) error { + return m.handler(ctx, imageRef, authConfig, out) +} + +func Test_pullImage_without_tag(t *testing.T) { + var s Server + + s.ImageMgr = &mockImgePull{ + ImageMgr: &mgr.ImageManager{}, + handler: func(ctx context.Context, imageRef string, authConfig *types.AuthConfig, out io.Writer) error { + assert.Equal(t, "reg.abc.com/base/os:7.2", imageRef) + return nil + }, + } + req := &http.Request{ + Form: map[string][]string{"fromImage": {"reg.abc.com/base/os:7.2"}}, + Header: map[string][]string{}, + } + s.pullImage(context.Background(), nil, req) +} +``` + +### 2.2.3 Summery + +ImageManager and RoundTripper are modeled in the same way except for the number of functions defined by the interface. In general, developers can manually define a structure that uses methods as fields, as shown in the next code. + +```bash +type Do interface { + Add(x int, y int) int + Sub(x int, y int) int +} + +type mockDo struct { + addFunc func(x int, y int) int + subFunc func(x int, y int) int +} + +// Add implements Do.Add function. +type (m *mockDo) Add(x int, y int) int { + return m.addFunc(x, y) +} + +// Sub implements Do.Sub function. +type (m *mockDo) Sub(x int, y int) int { + return m.subFunc(x, y) +} +``` + +When the interface is large and complex, the manual method will impose a test burden on the developer, so the community provides automatically generated tools, such as [mockery](https://github.com/vektra/mockery), to ease the burden on the developer. + +## 2.3 Other situation + +Sometimes it relies on third-party services, such as the PouchContainer client is a typical case. The Duck Type described above can complete the test of this case. In addition, we can also complete the request processing by registering the http.Handler and starting mockHTTPServer. This way of testing is relatively heavy, it is recommended to consider the use of the Duck Type test, or put it into the integration test. + +> NOTE: The golang community has done[monkeypatch](https://github.com/bouk/monkey) by modifying the binary code. This tool is not recommended, or it is recommended that developers design and write testable code. + +## 2.4 Summery + +PouchContainer integrates unit test cases into the code review phase, and reviewers can view the running of test cases at any time. + +# 3.Conclusion + +In the code review phase, code style checking, unit testing, and integration testing should be run through continuous integration to help reviewers make accurate decisions. Currently, PouchContainer checks code style, tests and executes other operations primarily through TravisCI/CircleCI and [pouchrobot](https://github.com/pouchcontainer/pouchrobot). + + +