CI/CD（基于 k8s） - Powered by MinDoc

前置条件

安装 k8s：
参见：使用 kuboard-spray 搭建 k8s 集群
安装 harbor：
参见：centos7 安装 harbor2.7.2
安装 gitlab：
参见：版本控制 – gitlab16.0.0
安装 jenkins：
参见：持续集成 – jenkins2.385
安装 efk：
参见：docker 安装 EFK6.4.3
安装 golang：
参见：源码安装 golang

配置并重启 containerd：
vim /etc/containerd/config.toml

 version = 2
 root = "/var/lib/containerd"
 state = "/run/containerd"
 oom_score = 0

 [grpc]
   max_recv_message_size = 16777216
   max_send_message_size = 16777216

 [debug]
   level = "info"

 [metrics]
   address = ""
   grpc_histogram = false

 [plugins]
   [plugins."io.containerd.grpc.v1.cri"]
     sandbox_image = "k8s.gcr.io/pause:3.8"
     max_container_log_line_size = -1
     [plugins."io.containerd.grpc.v1.cri".containerd]
       default_runtime_name = "runc"
       snapshotter = "overlayfs"
       [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
           runtime_type = "io.containerd.runc.v2"
           runtime_engine = ""
           runtime_root = ""

           [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
             systemdCgroup = true
     [plugins."io.containerd.grpc.v1.cri".registry]
       [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
         [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
           endpoint = ["https://registry-1.docker.io"]
         ####################################
         ####### 添加 harbor 相关配置 ########
         ####################################
         [plugins."io.containerd.grpc.v1.cri".registry.mirrors."xx.xx.xx.xx:9011"]
           endpoint = ["http://xx.xx.xx.xx:9011"]
       [plugins."io.containerd.grpc.v1.cri".registry.configs]
         [plugins."io.containerd.grpc.v1.cri".registry.configs."xx.xx.xx.xx:9011".tls]
           insecure_skip_verify = true
         [plugins."io.containerd.grpc.v1.cri".registry.configs."xx.xx.xx.xx:9011".auth]
           username = "admin"
           password = "admin123"

systemctl restart containerd

gitlab 项目代码

目录结构：

|—— myproject/   // 根目录
****|—— config/
********|—— config.yaml.example  // 配置文件样本
****|—— log/
********|—— info.log  // 日志文件（自动生成）
****|—— utils/  // 工具目录（go test 目录）
********|—— utils.go
********|—— utils_test.go
****|—— .gitignore  // git 配置忽略文件
****|—— Dockerfile  // docker 镜像构建脚本
****|—— deploy.yaml  // k8s 部署脚本
****|—— go.mod.example  // go.mod 模板文件
****|—— main.go  // 入口文件
****|—— Makefile  // makefile 文件

编写 myproject/config/config.yaml.example：
```
port: 8999
name: lee
```

编写 myproject/utils/utils.go：

 package utils

 func Add(a, b int) int {
     return a + b
 }

编写 myproject/utils/utils_test.go：

 package utils

 import (
     "testing"
 )

 func TestUtils(t *testing.T) {
     c := Add(1, 2)
     if c != 3 {
         t.Errorf("%d + %d != %d", 1, 2, 3)
     }
     t.Log("测试通过")
 }

编写 myproject/.gitignore：

log/info.log
go.sum
go.mod
config/config.yaml

编写 myproject/Dockerfile：

 # build
 FROM golang:1.19 AS build
 ENV GO111MODULE=on \
     GOPROXY=https://goproxy.cn,direct \
     CGO_ENABLED=0 \
     GOOS=linux \
     GOARCH=amd64
 COPY . /myapp
 WORKDIR /myapp
 RUN go mod tidy \
     && go build -v -ldflags "-w" -o /myapp/run main.go
 # deploy
 FROM alpine:3 AS deploy
 WORKDIR /myapp
 COPY --from=build /myapp/run ./run
 RUN chmod +x ./run
 CMD [ "./run" ]

编写 myproject/deploy.yaml（注意: 这里仅为了演示而用到了 ConfigMap，实际工作中请使用 etcd 或 consul 做配置中心）：

 apiVersion: v1
 kind: Namespace
 metadata:
     name: test-ns
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
     name: test-cm
     namespace: test-ns
 data:
     config.yaml: |
         port: 8999
         name: lee
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
     name: test-deploy
     namespace: test-ns
     labels:
         app: myproject
 spec:
     replicas: 1
     selector:
         matchLabels:
             app: myproject
     template:
         metadata:
             labels:
                 app: myproject
         spec:
             containers:
                 -   name: myproject-pod
                     image: __IMAGE__
                     imagePullPolicy: IfNotPresent
                     ports:
                         -   containerPort: 8999
                             name: t-d-p
                     volumeMounts:
                         -   name: test-deploy-volume-log
                             mountPath: /myapp/log
                         -   name: test-deploy-volume-cm
                             mountPath: /myapp/config
             volumes:
                 -   name: test-deploy-volume-log
                     hostPath:
                         path: /path/to/myproject/log
                 -   name: test-deploy-volume-cm
                     configMap:
                         name: test-cm
 ---
 apiVersion: v1
 kind: Service
 metadata:
     name: test-svc
     namespace: test-ns
 spec:
     selector:
         app: myproject
     type: ClusterIP
     ports:
         -   name: test-svc-port
             protocol: TCP
             port: 8999
             targetPort: t-d-p

编写 myproject/go.mod.example：
module myproject

编写 myproject/main.go（注意: 这里仅为了演示而用到了 ConfigMap，实际工作中请使用 etcd 或 consul 做配置中心）：

 package main

 import (
     "fmt"
     "github.com/fsnotify/fsnotify"
     "github.com/sirupsen/logrus"
     "github.com/spf13/viper"
     "myproject/utils"
     "net/http"
     "os"
     "strconv"
 )

 func init() {
     // 初始化 logrus
     logrus.SetReportCaller(true)
     logrus.SetFormatter(&logrus.TextFormatter{
         TimestampFormat: "2006-01-02 15:03:04",
     })
     file, err := os.OpenFile("./log/info.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
     if err == nil {
         logrus.SetOutput(file)
     } else {
         logrus.Fatalf("输出到日志文件失败：%s \n", err)
     }
     logrus.SetLevel(logrus.TraceLevel)
     // 加载配置文件
     loadConfig()
 }

 func loadConfig() {
     viper.SetConfigName("config.yaml")
     viper.SetConfigType("yaml")
     viper.AddConfigPath("./config")
     if err := viper.ReadInConfig(); err != nil {
         logrus.Fatalf("获取配置文件失败： %s \n", err)
     }
 }

 func watchConfig() {
     watcher, _ := fsnotify.NewWatcher()
     defer watcher.Close()
     done := make(chan bool)
     go func() {
         for {
             select {
             case _, ok := <-watcher.Events:
                 if !ok {
                     return
                 }
                 loadConfig()
                 logrus.Info("配置文件有变化，当前 【name】 值为：", viper.GetString("name"))
             }
         }
     }()
     err := watcher.Add("./config/config.yaml")
     if err != nil {
         fmt.Println("监控文件/文件夹失败：", err)
     }
     <-done
 }

 func myHandler(w http.ResponseWriter, r *http.Request) {
     r.ParseForm()
     var a, b string
     params := r.Form
     if len(params["a"]) > 0 {
         a = params["a"][0]
     }
     if len(params["b"]) > 0 {
         b = params["b"][0]
     }
     aInt, _ := strconv.Atoi(a) // 参数 a
     bInt, _ := strconv.Atoi(b) // 参数 b
     res := utils.Add(aInt, bInt)
     output := fmt.Sprintf("%d + %d = %d", aInt, bInt, res)
     logrus.WithFields(logrus.Fields{
         "a": aInt,
         "b": bInt,
     }).Info(output)
     fmt.Fprintln(w, output)
 }

 func main() {
     go watchConfig() // 监听 ConfigMap 的变化，进行配置重载
     port := viper.GetString("port")
     addr := fmt.Sprintf(":%s", port)
     http.HandleFunc("/", myHandler)
     svc := http.Server{
         Addr:    addr,
         Handler: http.DefaultServeMux,
     }
     fmt.Printf("http 服务器启动成功，访问地址为：%s \n", addr)
     if err := svc.ListenAndServe(); err != nil {
         logrus.Fatalf("启动 http 服务失败：%s \n", err)
     }
 }

编写 myproject/Makefile：

# 所有自定义步骤列表
.PHONY: all test build push replace deploy recover help

# 自定义变量
OUTPUT_FILE=myapp
IMAGE_NAME=myproject
IMAGE_VERSION=v1.0
HARBOR_URL=xx.xx.xx.xx:9011
HARBOR_USERNAME=admin
HARBOR_PASSWORD=admin123

# 直接运行 make 命令调用的步骤
all: test build push replace deploy recover

# 测试
test:
    cd utils && \
    go test

# 构建
build:
    docker build -t ${HARBOR_URL}/library/${IMAGE_NAME}:${IMAGE_VERSION} .

# 推送
push:
    docker login -u ${HARBOR_USERNAME} -p ${HARBOR_PASSWORD} ${HARBOR_URL}
    docker push ${HARBOR_URL}/library/${IMAGE_NAME}:${IMAGE_VERSION}

# 替换版本
replace:
    sed -i "s/__IMAGE__/${HARBOR_URL}\/library\/${IMAGE_NAME}:${IMAGE_VERSION}/" deploy.yaml

# 部署（运行）
deploy:
    kubectl apply -f deploy.yaml

# 恢复替换版本
recover:
    sed -i "s/${HARBOR_URL}\/library\/${IMAGE_NAME}:${IMAGE_VERSION}/__IMAGE__/" deploy.yaml

# 帮助信息（"@"屏蔽输出）
help:
    @echo "【make】 -- 测试、构建、部署"
    @echo "【make test】 -- 测试"
    @echo "【make build】 -- 构建"
    @echo "【make push】 -- 推送"
    @echo "【make replace】 -- 替换版本"
    @echo "【make deploy】 -- 部署"
    @echo "【make recover】 -- 恢复替换版本"
    @echo "【make help】 -- 查看帮助"

配置文件：

jenkins pipeline：

pipeline{
 //agent any
 agent {
     label 'master'  // 指定在 master 节点运行
 }
 stages{
     stage("pull"){
         steps{
             sh """
                 cd /path/to/myproject
                 git checkout .
                 git reset --hard
                 git stash
                 git pull
             """
         }
     }
     stage("tidy"){
         steps{
             sh """
                 go env -w GO111MODULE=on
                 go env -w GOPROXY=https://goproxy.cn,direct
                 cd /path/to/myproject
                 cp go.mod.example go.mod
                 go mod tidy
             """
         }
     }
     stage("deploy"){
         steps{
             sh """
                 cd /path/to/myproject
                 make
             """
         }
     }
     stage("prune"){
         steps{
             sh """
                 docker image prune -f
             """
         }
     }
 }
}

efk：
cat /path/to/filebeat/config/filebeat.yml

 filebeat.inputs:
     -   type: log
         enabled: true
         paths:
             - /var/myproject/log/*.log
         fields:
             log_topics: "myproject"

 setup.template.settings:
     index.number_of_shards: 3

 setup.template.name: "filebeat*"
 setup.template.pattern: "filebeat*"

 output.elasticsearch:
     hosts: [ "elasticsearch643:9200" ]
     index: "project-other"
     indices:
         -   index: "myproject"
             when.contains:
                 fields:
                     log_topics: "myproject"

各服务器访问地址

harbor：
xx.xx.xx.xx:9011
gitlab：
xx.xx.xx.xx:8888
jenkins：
xx.xx.xx.xx:9999
myproject：
xx.xx.xx.xx:8999
kibana：
xx.xx.xx.xx:5601

文档更新时间: 2024-04-20 10:57 作者：lee