1. 单台虚拟机部署3个tomcat多实例jpress-v3.0.6

1.1 解压tomcat包

1
[root@tomcat tomcat]# tar xf /app/tools/apache-tomcat-8.5.50.tar.gz

1.2 复制3个tomcat实例

1
2
3
[root@tomcat tomcat]# cp -r apache-tomcat-8.5.50/ tomcat_8081
[root@tomcat tomcat]# cp -r apache-tomcat-8.5.50/ tomcat_8082
[root@tomcat tomcat]# cp -r apache-tomcat-8.5.50/ tomcat_8083

1.3 修改3个tomcat实例的端口，进行区分

1
2
3
4
5
6
7
8
9
[root@tomcat tomcat]# sed -i.bak 's#8080#8081#g' tomcat_8081/conf/server.xml
[root@tomcat tomcat]# sed -i.bak 's#8005#8006#g' tomcat_8081/conf/server.xml
[root@tomcat tomcat]# sed -i.bak 's#8009#8010#g' tomcat_8081/conf/server.xml
[root@tomcat tomcat]# sed -i.bak 's#8080#8082#g' tomcat_8082/conf/server.xml
[root@tomcat tomcat]# sed -i.bak 's#8005#8007#g' tomcat_8082/conf/server.xml
[root@tomcat tomcat]# sed -i.bak 's#8009#8011#g' tomcat_8082/conf/server.xml
[root@tomcat tomcat]# sed -i.bak 's#8080#8083#g' tomcat_8083/conf/server.xml
[root@tomcat tomcat]# sed -i.bak 's#8005#8008#g' tomcat_8083/conf/server.xml
[root@tomcat tomcat]# sed -i.bak 's#8009#8012#g' tomcat_8083/conf/server.xml

1.4 将3个tomcat实例包移动到/app下

1
[root@tomcat tomcat]# mv tomcat_808* /app/

1.5 启动3个tomcat实例

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
[root@tomcat tomcat]# /app/tomcat_8081/bin/startup.sh 
Using CATALINA_BASE:   /app/tomcat_8081
Using CATALINA_HOME:   /app/tomcat_8081
Using CATALINA_TMPDIR: /app/tomcat_8081/temp
Using JRE_HOME:        /app/jdk
Using CLASSPATH:       /app/tomcat_8081/bin/bootstrap.jar:/app/tomcat_8081/bin.jar
Tomcat started.
[root@tomcat tomcat]# /app/tomcat_8082/bin/startup.sh 
Using CATALINA_BASE:   /app/tomcat_8082
Using CATALINA_HOME:   /app/tomcat_8082
Using CATALINA_TMPDIR: /app/tomcat_8082/temp
Using JRE_HOME:        /app/jdk
Using CLASSPATH:       /app/tomcat_8082/bin/bootstrap.jar:/app/tomcat_8082/bin.jar
Tomcat started.
[root@tomcat tomcat]# /app/tomcat_8083/bin/startup.sh 
Using CATALINA_BASE:   /app/tomcat_8083
Using CATALINA_HOME:   /app/tomcat_8083
Using CATALINA_TMPDIR: /app/tomcat_8083/temp
Using JRE_HOME:        /app/jdk
Using CLASSPATH:       /app/tomcat_8083/bin/bootstrap.jar:/app/tomcat_8083/bin.jar
Tomcat started.

1.6 查看3个tomcat实例的端口

1
2
3
4
5
6
7
8
9
10
[root@tomcat webapps]# ss -lntup | grep java
tcp    LISTEN     0      1      ::ffff:127.0.0.1:8007                 :::*                   users:(("java",pid=58100,fd=79))
tcp    LISTEN     0      1      ::ffff:127.0.0.1:8008                 :::*                   users:(("java",pid=58157,fd=79))
tcp    LISTEN     0      100      :::8010                 :::*                   users:(("java",pid=58041,fd=58))
tcp    LISTEN     0      100      :::8011                 :::*                   users:(("java",pid=58100,fd=58))
tcp    LISTEN     0      100      :::8012                 :::*                   users:(("java",pid=58157,fd=58))
tcp    LISTEN     0      100      :::8081                 :::*                   users:(("java",pid=58041,fd=53))
tcp    LISTEN     0      100      :::8082                 :::*                   users:(("java",pid=58100,fd=53))
tcp    LISTEN     0      100      :::8083                 :::*                   users:(("java",pid=58157,fd=53))
tcp    LISTEN     0      1      ::ffff:127.0.0.1:8006                 :::*                   users:(("java",pid=58041,fd=79))

1.7 分别部署3个tomcat实例的jpress-v3.0.6

软件包存放路径为：/app/tomcat_8081/webapps/

1.8 远程数据库部署

1
2
3
4
5
6
7
# 创建3个数据库
mysql> create database jpress_8081;
mysql> create database jpress_8082;
mysql> create database jpress_8083;

# 创建用户jpress并授权
mysql> grant all on *.* to jpress@'%' identified by '123456';

2. 故障案例：tomcat负载高排查方法

2.1 找出问题进程

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# 利用top命令找到CPU/MEM（根据实际需求选择）找到负载最高的进程，并记录PID号，如下例中则为58100
[root@tomcat ~]# top 
top - 19:50:00 up 16:26,  2 users,  load average: 0.02, 0.04, 0.05
Tasks: 106 total,   1 running, 105 sleeping,   0 stopped,   0 zombie
%Cpu(s):  0.2 us,  0.0 sy,  0.0 ni, 99.8 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  2028116 total,    64292 free,  1338380 used,   625444 buff/cache
KiB Swap:        0 total,        0 free,        0 used.   472492 avail Mem 

   PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND      
 58100 root      20   0 3159080 354528  13684 S   0.3 17.5   0:22.81 java         
     1 root      20   0  125972   4468   2620 S   0.0  0.2   0:17.71 systemd      
     2 root      20   0       0      0      0 S   0.0  0.0   0:00.01 kthreadd     
     3 root      20   0       0      0      0 S   0.0  0.0   0:01.62 ksoftirqd/0  
     5 root       0 -20       0      0      0 S   0.0  0.0   0:00.00 kworker/0:0H 
     7 root      rt   0       0      0      0 S   0.0  0.0   0:00.29 migration/0  
     8 root      20   0       0      0      0 S   0.0  0.0   0:00.00 rcu_bh       
     9 root      20   0       0      0      0 S   0.0  0.0   0:04.11 rcu_sched    
    10 root       0 -20       0      0      0 S   0.0  0.0   0:00.00 lru-add-drain

2.2 通过进程PID找出是哪个java问题线程

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
# 利用top -Hp  PID 查找负载最高的线程，如下例中的58111
[root@tomcat ~]# top -Hp 58100
top - 19:53:13 up 16:29,  2 users,  load average: 0.00, 0.02, 0.05
Threads:  85 total,   0 running,  85 sleeping,   0 stopped,   0 zombie
%Cpu(s):  1.2 us,  0.3 sy,  0.0 ni, 98.3 id,  0.0 wa,  0.0 hi,  0.2 si,  0.0 st
KiB Mem :  2028116 total,    73444 free,  1345008 used,   609664 buff/cache
KiB Swap:        0 total,        0 free,        0 used.   466180 avail Mem 

   PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND                             
 58111 root      20   0 3159080 354588  13684 S  0.7 17.5   0:02.33 java                                
 58100 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.00 java                                
 58101 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.55 java                                
 58102 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.25 java                                
 58103 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.23 java                                
 58104 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.62 java                                
 58105 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.01 java                                
 58106 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.01 java                                
 58107 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.00 java                                
 58108 root      20   0 3159080 354588  13684 S  0.0 17.5   0:06.92 java                                
 58109 root      20   0 3159080 354588  13684 S  0.0 17.5   0:01.58 java                                
 58110 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.00 java                                
 58112 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.20 java                                
 58113 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.00 java                                
 58114 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.13 java                                
 58115 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.13 java                                
 58119 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.37 java                                
 58130 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.16 java                                
 58131 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.13 java                                
 58132 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.07 java                                
 58133 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.12 java                                
 58144 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.15 java                                
 58145 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.14 java                                
 58146 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.00 java                                
 58147 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.13 java                                
 60193 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.00 java                                
 60196 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.00 java                                
 60198 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.00 java                                
 60201 root      20   0 3159080 354588  13684 S  0.0 17.5   0:00.02 java

2.3 将问题线程的pid号转换为16进制

1
2
[root@tomcat ~]# echo 'obase=16;58111' | bc
E2FF

2.4 利用jstack配合grep过滤出并显示问题进程的详细信息

1
2
3
4
# jstack 问题进程PID | grep -i 问题线程PID（16进制数）
[root@tomcat ~]# jstack 58100 | grep -i E2FF
"VM Periodic Task Thread" os_prio=0 tid=0x00007efc3c0c5800 nid=0xe2ff waiting on condition
说明：筛选出的信息可以提供开发人员查看排查问题

2.5 利用jmap显示对应PID的jvm内存使用情况

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
[root@tomcat ~]# jmap -heap 58100
Attaching to process ID 58100, please wait...
Debugger attached successfully.
Server compiler detected.
JVM version is 25.241-b07

using thread-local object allocation.
Parallel GC with 2 thread(s)

Heap Configuration:
   MinHeapFreeRatio         = 0
   MaxHeapFreeRatio         = 100
   MaxHeapSize              = 520093696 (496.0MB)
   NewSize                  = 11010048 (10.5MB)
   MaxNewSize               = 173015040 (165.0MB)
   OldSize                  = 22544384 (21.5MB)
   NewRatio                 = 2
   SurvivorRatio            = 8
   MetaspaceSize            = 21807104 (20.796875MB)
   CompressedClassSpaceSize = 1073741824 (1024.0MB)
   MaxMetaspaceSize         = 17592186044415 MB
   G1HeapRegionSize         = 0 (0.0MB)

Heap Usage:
PS Young Generation
Eden Space:
   capacity = 125304832 (119.5MB)
   used     = 61121864 (58.29035186767578MB)
   free     = 64182968 (61.20964813232422MB)
   48.77853712776216% used
From Space:
   capacity = 23068672 (22.0MB)
   used     = 10506912 (10.020172119140625MB)
   free     = 12561760 (11.979827880859375MB)
   45.54623690518466% used
To Space:
   capacity = 22544384 (21.5MB)
   used     = 0 (0.0MB)
   free     = 22544384 (21.5MB)
   0.0% used
PS Old Generation
   capacity = 35651584 (34.0MB)
   used     = 13678312 (13.044654846191406MB)
   free     = 21973272 (20.955345153808594MB)
   38.36663190056296% used

25267 interned Strings occupying 2605064 bytes.

2.6 导出jvm内存内容

1
2
3
4
5
[root@tomcat ~]# jmap -dump:format=b,file=/root/tomcat.bin 58100
Dumping heap to /root/tomcat.bin ...
File exists
[root@tomcat ~]# ll /root/tomcat.bin 
-rw------- 1 root root 69961630 Feb 10 17:48 /root/tomcat.bin

2.7 导出的文件可以用MemoryAnalyzer打开进行分析