Jenkins 集成 Harbor、Helm 上线PHP、NODE项目

Docker-compose 部署 Jenkins

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
# 可以使用 docker-compose 安装
mkdir -p /data/jenkins.csm.io
cd /data/jenkins.csm.io && mkdir data
vim docker-compose.yml

version: "3.9"
services:
  jenkins:
    image: jenkins/jenkins:lts
    container_name: jenkins
    hostname: jenkins.csm.io
    user: root
    privileged: true
    environment:
      TZ: 'Asia/Shanghai'
      # 设置 Java 参数
      JAVA_OPTS: |
        -Dorg.apache.catalina.security.SecurityListener.UMASK=`umask`
        -Djava.awt.headless=true
        -Xms512m
        -Xmx1024m
        -XX:PermSize=256M
        -XX:MaxPermSize=512M
    volumes:
      - ./data:/var/jenkins_home
      - /usr/lib64/libltdl.so.7:/usr/lib/x86_64-linux-gnu/libltdl.so.7
      - /var/run/docker.sock:/var/run/docker.sock
      - /usr/bin/docker:/usr/bin/docker
    ports:
      - 80:8080
    restart: always

# 1、按推荐插件进行安装，进入界面下载 Publish Over SSH 插件（构建后操作，使用helm）
# 2、进入系统配置，在 Publish over SSH 中，添加 SSH server （k8s-master节点，并用 helm 部署其中）

docker-compose up -d

免密登录至测试服务器

1
2
3
4
5
6
7
8
9
10
# 产生公私密钥对（已经生成了）
ssh-keygen

# 将本地的公钥上传到远程服务器
使用 ssh-copy-id username@your-server-ip 命令

# 输入远程服务器的密码
xxxxxx

# 现在使用 ssh username@your-server-ip 命令即可免密登录了

做下推送镜像测试：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# Jenkins 端配置推送镜像,(上线要将k8s所有节点配置下列所有信息)
vim /etc/hosts
192.168.2.82  harbor.testcsm.com
192.168.2.82  harbor

vim /etc/docker/daemon.json
{
  "insecure-registries": ["http://harbor.testcsm.com"],
  "registry-mirrors": ["https://k2eqx0n7.mirror.aliyuncs.com"]
}

# 给镜像打标签
docker tag 镜像名称:版本号 IP地址/仓库名称/镜像名称:版本号
docker tag SOURCE_IMAGE[:TAG] TARGET_IMAGE[:TAG]

# 登录 harbor
docker login -u admin -p Harbor12345 harbor.testcsm.com

# 推送镜像
docker push harbor.testcsm.com/test/api:1

Dockerfile 制作基础镜像

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
# docker build -t csm.io/php:apache-composer .
#FROM php:apache-bullseye as phpbase
FROM php:8.0-apache-bullseye as phpbase
COPY sources.list /etc/apt/sources.list

RUN apt-get update && apt-get upgrade -y \
    && apt-get install -y libpng-dev libjpeg-dev libxpm-dev libfreetype6-dev libzstd-dev \
    # intl扩展
    && apt-get install -y libicu-dev \
    # tools
    && apt-get install -y apt-utils \
    # tools and cron
    && apt-get install -y cron vim wget curl git sudo zip unzip\
    # timezone
    && apt-get install -y tzdata \
    && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime\
    # rsyslog
    && apt-get install -y rsyslog \
    && sed -i 's/#cron/cron/g' /etc/rsyslog.conf \
    && service rsyslog restart \
    # mkdir log
    && mkdir -p /var/www/csm/runtime/log \
    && mkdir /var/www/csm/runtime/session \
    && mkdir /var/www/csm/runtime/tmp \
    && chown -R www-data. /var/www/csm/runtime/log
    
# composer
RUN cd /tmp \
    && wget https://getcomposer.org/composer.phar \
    && mv composer.phar composer \
    && chmod +x composer  \
    && mv composer /usr/local/bin \
    && composer config -g repo.packagist composer https://mirrors.aliyun.com/composer/

RUN docker-php-source extract \
    && docker-php-ext-configure gd --with-jpeg --with-freetype --with-xpm --enable-gd-jis-conv \
     # intl扩展
    && docker-php-ext-configure intl \
    && docker-php-ext-install intl \
    && docker-php-ext-install pdo_mysql gd sockets bcmath \
    && docker-php-source delete

RUN yes yes | pecl install igbinary redis \
    && docker-php-ext-enable igbinary redis \
    && ln -s /etc/apache2/mods-available/rewrite.load /etc/apache2/mods-enabled/rewrite.load

# open max files
RUN echo "* soft nofile 65535"  >> /etc/security/limits.conf \
    && echo "* hard nofile 65535"  >> /etc/security/limits.conf \
    && echo "fs.file-max = 6553560" >> /etc/sysctl.conf \
    && apt autoremove

Dockerfile 编写业务镜像

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
# api
FROM csm.io/php:apache-composer

RUN  mkdir -p /var/www/csm/runtime/log && mkdir  /var/www/csm/runtime/session && mkdir /var/www/csm/runtime/tmp && chown -R www-data. /var/www/csm/runtime/


WORKDIR /var/www/csm

# 时区 /etc/timezone 没有挂载
ADD ./test-api.tar.gz  /var/www/csm
ADD ./\.env  /var/www/csm
ADD ./000-default.conf  /etc/apache2/sites-enabled/000-default.conf
RUN composer update

CMD [ "bash", "/var/www/csm/crontab/startup.sh" ]

#---------------------------------------------------------------------------
# admin-api
FROM csm.io/php:apache-composer

RUN  mkdir -p /var/www/csm/runtime/log && mkdir  /var/www/csm/runtime/session && mkdir /var/www/csm/runtime/tmp && chown -R www-data. /var/www/csm/runtime/


WORKDIR /var/www/csm

# 时区 /etc/timezone 没有挂载
ADD ./test-admin-api.tar.gz  /var/www/csm
ADD ./\.env  /var/www/csm
ADD ./php.ini-development /usr/local/etc/php/php.ini
ADD ./000-default.conf  /etc/apache2/sites-enabled/000-default.conf
ADD ./crontab/startup.sh:/var/www/csm/crontab/startup.sh
RUN composer update

CMD [ "bash", "/var/www/csm/crontab/startup.sh" ]

免密登录至测试服务器

1
2
3
4
5
6
7
8
9
10
# 产生公私密钥对（已经生成了）
ssh-keygen

# 将本地的公钥上传到远程服务器
使用 ssh-copy-id username@your-server-ip 命令

# 输入远程服务器的密码
xxxxxx

# 现在使用 ssh username@your-server-ip 命令即可免密登录了

编写 Shell

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
#-----------------------------------------------
cd /var/jenkins_home/workspace/$JOB_NAME/  # 进入源代码目录 
rm -f $JOB_NAME.tar.gz &&  tar zcvf $JOB_NAME.tar.gz  ./*
# 构建镜像
docker build -f ./Dockerfile -t harbor.testcsm.com/test/api:v$BUILD_ID .

docker login -u admin -p Harbor12345 harbor.testcsm.com
docker push harbor.testcsm.com/test/api:v$BUILD_ID
docker rmi harbor.testcsm.com/test/api:v$BUILD_ID

ssh 192.168.2.93 "\
  cp /opt/helmchart/test-api/templates/deployment.bak /opt/helmchart/test-api/templates/deployment.yaml
  sed -i "s/tag-version/v$BUILD_ID/g" /opt/helmchart/test-api/templates/deployment.yaml 
"

ssh 192.168.2.93 " cd /opt/helmchart &&  helm upgrade test-api  test-api  --set name=test-api"

ssh 192.168.2.93 " kubectl apply -f  /root/test-api-ingress.yaml "

#--------------
# 发送机器人消息
# 获得上一次提交的时间
PREVIOUS_COMMIT_TIME=`git show $GIT_PREVIOUS_COMMIT | grep 'Date: ' | awk -F 'Date:  ' '{print $2}'`
# 移除上一次提交的备注信息
PREVIOUS_COMMIT_TIME=`echo $PREVIOUS_COMMIT_TIME | sed 's/+0800/+075959/'`
# 获得上一次提交的日志并保存到文件
echo 【测试环境 - 总管后台 api 】部署成功，更新如下： > CHANGELOG.log
git log --since "$PREVIOUS_COMMIT_TIME" | grep $GIT_PREVIOUS_COMMIT -B 100 >> CHANGELOG.log
tailLen=`wc -l CHANGELOG.log | awk '{print $1}'`
changelog=/data/jenkins.csm.io/data/workspace/$JOB_NAME/CHANGELOG.log


# 发送消息
ssh 192.168.2.83 "head -n $(($tailLen-1)) $changelog | /data/qiye-wechat-notice f24d81ed-619e-4604-9729-78543fd5dde8"

Helm 做测试

1
2
3
4
5
6
7
8
9
10
helm repo add stable http://mirror.azure.cn/kubernetes/charts   #微软的很全推荐

# 从仓库下载charts
helm pull stable/nginx-ingress
# 解压charts
tar xf nginx-ingress-1.41.3.tgz

# 安装-卸载
helm install nginx-ingress nginx-ingress -n kube-system
helm uninstall nginx-ingress nginx-ingress -n kube-system

Helm Chart 编写上线模板

1
2
3
4
5
6
7
8
# 举个例子先：
helm install $JOB_NAME \
   --set name=$JOB_NAME \
   --set image=harbor.testcsm.com/test/php:v$BUILD_ID

#--set image=harbor.testcsm.com/test/php:v7
helm install test-api  test-api  --set name=test-api

helm 目录结构
1
2
3
4
5
6
7
8
9
10
└── test-api
    ├── charts
    ├── Chart.yaml
    ├── templates
    │   ├── deployment.bak
    │   ├── deployment.yaml
    │   ├── ingress.bak
    │   └── service.yaml
    └── values.yaml

Chart
1
2
3
4
5
6
apiVersion: v2
name: test-api
description: A Helm chart for Kubernetes
type: application
version: 0.1.0
appVersion: 1.20.2
values
1
2
3
4
5
6
7
8
9
name:
proxyname:
image:
tag:
namespace: default
replicas: 1
targetPort: 80
servicePort: 9501
#servicePort: 80
Deployment
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: {{ .Values.name }}
  name: {{ .Values.name }}
  namespace: {{ .Values.namespace }}
spec:
  replicas: {{ .Values.replicas }}
  selector:
    matchLabels:
      app: {{ .Values.name }}
  template:
    metadata:
      labels:
        app: {{ .Values.name }}
    spec:
      containers:
      #- image: {{ .Values.image }}:{{ .Values.tag }}
      - image: harbor.testcsm.com/test/api:v8
        imagePullPolicy: "Always"
        workingDir: /var/www/csm
        name: {{ .Values.name }}
        resources:
          limits:
             cpu: "2"
             memory: 2048Mi
          requests:
             cpu: "0.2"
             memory: 1024Mi
Service
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
apiVersion: v1
kind: Service
metadata:
  labels:
    app: {{ .Values.name }}
  name: {{ .Values.name }}
  namespace: {{ .Values.namespace }}
spec:
  ports:
  - port: {{ .Values.targetPort }}
    protocol: TCP
    targetPort: {{ .Values.targetPort }}
  selector:
    app: {{ .Values.name }}
  type: NodePort

Ingress 编写

以 Api 项目为例
1
kubectl create secret tls tls-test-api --key test.api.huosanyun.com.key --cert test.api.huosanyun.com.pem
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: test-api
spec:
  rules:
  - host: test.api.huosanyun.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: test-api
            port:
              number: 80
  tls:
  - hosts:
    - test.api.huosanyun.com
    secretName: tls-test-api

Jenkins Node 环境安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
# 安装源
mv /etc/apt/sources.list  /etc/apt/sources.list.back
echo "" > /etc/apt/sources.list 
cat <<EOF > /etc/apt/sources.list 
deb http://mirrors.cloud.tencent.com/debian/ bullseye main non-free contrib
deb http://mirrors.cloud.tencent.com/debian-security bullseye/updates main
deb http://mirrors.cloud.tencent.com/debian/ bullseye-updates main non-free contrib
deb http://mirrors.cloud.tencent.com/debian/ bullseye-backports main non-free contrib

deb-src http://mirrors.cloud.tencent.com/debian-security bullseye/updates main
deb-src http://mirrors.cloud.tencent.com/debian/ bullseye main non-free contrib
deb-src http://mirrors.cloud.tencent.com/debian/ bullseye-updates main non-free contrib
deb-src http://mirrors.cloud.tencent.com/debian/ bullseye-backports main non-free contrib
EOF

# 通过 apt 在终端中运行以下命令来确保系统是最新
sudo apt update
sudo apt upgrade

# 使用 PPA 安装 NodeJS
# 将 PPA 存储库添加到您的系统
curl -fsSL https://deb.nodesource.com/setup_17.x | sudo -E bash -

# 更新源列表以包含刚刚使用以下命令添加的新存储库
apt update -y

# 使用以下命令安装 NodeJS 和 NPM
apt install nodejs -y

# 验证版本
$ node -v
v17.4.0
$ npm -v
8.3.1