scd_tools_go/service/monitor/dataMonitor.go

package monitor

import (
	"fmt"
	"log"
	"math"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"scd_check_tools/conf"
	Db "scd_check_tools/db"
	"scd_check_tools/email"
	"scd_check_tools/global"
	"scd_check_tools/logger"
	"scd_check_tools/models/bo"
	"scd_check_tools/sms"
	"scd_check_tools/tools"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/shirou/gopsutil/cpu"
	"github.com/shirou/gopsutil/disk"

	//"github.com/shirou/gopsutil/load"
	"github.com/shirou/gopsutil/mem"

	"github.com/astaxie/beego/orm"
)

type DataMonitor struct {
	performanceRuntimeMonitorState  bool
	performanceRuntimeMonitorResult map[string]interface{}
	pmResultLock                    sync.RWMutex
}

var SignDataMonitor = new(DataMonitor)

var DataMonitorMsgChan chan map[string]interface{}

//设备数据处理监控
func StartDataMonitor() {
	DataMonitorMsgChan = make(chan map[string]interface{}, 10)
	sendMsg := func() {
		log.Println("启动短信发送进程")
		for {
			msgdata, ok := <-DataMonitorMsgChan
			if ok {
				if msgdata != nil {
					smsplatform := conf.GlobalConfig["sms.platform"]
					if smsplatform == "" {
						log.Println("未配置短信消息发送平台")
					} else {
						smsSend := sms.GetSmsInstance(smsplatform)
						datatype := msgdata["datatype"].(string)
						params := msgdata["params"].([]string)
						phone := msgdata["phone"].([]string)
						if conf.GlobalConfig["loglevel"] == "5" {
							log.Println(fmt.Sprintf("%v", msgdata))
						}
						_, err := smsSend.Send(datatype, params, phone)
						if err != nil {
							log.Println(err)
						}
					}
				}
			}
		}
	}
	go sendMsg()
	go dataTimoutMonitorStart()
	go dataAlterMonitorStart()
	go performanceMonitorStart()
	go attachmentMonitorStart()
	go StartClearTempScd()
	go StateOptimizeTable()
	go func() {
		for {
			select {
			case v := <-global.PerformanceRuntimeMonitorChan:
				log.Println(fmt.Sprintf("=======================性能监测标识：%d", v))
				if v == 1 {
					go SignDataMonitor.PerformanceRuntimeMonitorStart()
				} else {
					go SignDataMonitor.PerformanceRuntimeMonitorStop()
				}
			}
		}
		log.Println(fmt.Sprintf("=======================性能监测终止"))
	}()
}

//启动每周数据表优化进程
func StateOptimizeTable() {
	log.Println("启动每周数据表优化进程")
	for {
		nowDay, _ := strconv.Atoi(time.Now().Format("02"))
		nowHour := time.Now().Format("15")
		//每5天优化一次
		dbOptimizeMonitorIntervalDay, _ := bo.GetSysParamValue("dbOptimizeMonitorInterval", "5")
		dbOptimizeMonitorIntervalDayI, _ := strconv.Atoi(dbOptimizeMonitorIntervalDay)
		if nowHour != "02" || (nowDay%dbOptimizeMonitorIntervalDayI != 0) {
			time.Sleep(time.Hour * 1)
			continue
		}
		m := new(DataMonitor)
		m.OptimizeTable()
		time.Sleep(time.Hour * 1)
	}
}

//启动定时清理临时SCD数据进程
//每小时清理一次
func StartClearTempScd() {
	logger.Logger.Println("启动定时(每小时)清理临时SCD数据进程")
	for {
		m := new(DataMonitor)
		m.tempScdClear()
		time.Sleep(time.Minute * 60 * 24)
	}
}

//数据采集超时处理进程启动
func dataTimoutMonitorStart() {
	log.Println("启动数据超时检查处理进程")
}

//设备告警监测进程启动
func dataAlterMonitorStart() {
	log.Println("启动数据告警监测处理进程")
}

//常规性能监控.每1分钟监测一次
func performanceMonitorStart() {
	log.Println("启动性能监控处理进程")
	var err error
	osname := string(runtime.GOOS)
	cpuHC, _ := strconv.Atoi(tools.IsEmpty(conf.GlobalConfig["alarm_cpu_value"], "90"))    //cpu高耗用率
	memHC, _ := strconv.Atoi(tools.IsEmpty(conf.GlobalConfig["alarm_mem_value"], "90"))    //内存高耗用率
	diskHCC, _ := strconv.Atoi(tools.IsEmpty(conf.GlobalConfig["alarm_disk_value"], "90")) //C磁盘高耗用率
	diskHCI := diskHCC                                                                     //程序所在磁盘高耗用率
	cpuHC_lst := map[string]int{"ishc": 0, "hcnt": 0, "lcnt": 0}
	memHC_lst := map[string]int{"ishc": 0, "hcnt": 0, "lcnt": 0}
	diskHCC_lst := map[string]int{"ishc": 0, "hcnt": 0, "lcnt": 0}
	diskHCI_lst := map[string]int{"ishc": 0, "hcnt": 0, "lcnt": 0}
	thisIp, _ := tools.LocalIPv4()
	if thisIp != "" {
		thisIp = "[" + thisIp + "]"
	}
	appname := conf.GlobalConfig["appname"]
	//fmt.Println("======================IP：" + thisIp)
	db := orm.NewOrm()
	logalarming := false
	memalarming := false
	cpualarming := false
	diskalarming := false
	diskcalarming := false //c盘空间
	for {
		log_alarm_email, _ := bo.GetSysParamValue("log_alarm_email", "")
		go func() {
			//日志容量监测
			sql := "select round(sum(DATA_LENGTH/1024/1024),2) v from information_schema.tables where table_schema=? AND table_name='global_syslog'"
			spaceRowset := []orm.Params{}
			_, dberr := db.Raw(sql, Db.DbCfg.Db).Values(&spaceRowset)
			if dberr != nil {
				logger.Logger.Error(dberr)
			}
			if len(spaceRowset) > 0 {
				logspce, _ := strconv.Atoi(tools.IsEmpty(spaceRowset[0]["v"]))
				if logspce > 0 {
					vmod, _ := bo.GetSysParamValue("log_size_max", "0")
					logmax, _ := strconv.Atoi(vmod)
					logalarm1, _ := bo.GetSysParamValue("log_size_alarm1", "0")
					logalarmv1, _ := strconv.Atoi(logalarm1)
					logalarm2, _ := bo.GetSysParamValue("log_size_alarm2", "0")
					logalarmv2, _ := strconv.Atoi(logalarm2)
					if logmax > 0 {
						desc := ""
						if logalarmv2*logmax <= logspce {
							//告警
							desc = fmt.Sprintf("%s日志告警：当前系统日志容量已达 %d MB", appname, logspce)
						} else if logalarmv1*logmax <= logspce {
							//预警
							desc = fmt.Sprintf("%s日志预警：当前系统日志容量已达 %d MB", appname, logspce)
						}
						if !logalarming && desc != "" {
							new(bo.SystemAlarm).Save("1", "log", desc, 0)
							logalarming = true
						} else if logalarming && desc == "" {
							desc = appname + "日志容量预（告）警已解除"
							new(bo.SystemAlarm).Save("1", "log", desc, 1)
							logalarming = false
						}
						if desc != "" {
							if log_alarm_email != "" {
								err := new(email.Send).SendEmail(log_alarm_email, appname+"日志容量预(告)警", desc)
								if err != nil {
									logger.Logger.Error(err)
								}
							}
						}
					}
				}
			}
		}()
		sql := "select mobilephone from t_data_user u,global_const_code t where u.role=t.id and t.code='role_sysadmin'"
		ps := []orm.Params{}
		_, err = db.Raw(sql).Values(&ps)
		if err != nil {
			log.Println(err)
			time.Sleep(5 * time.Minute)
			continue
		}
		devicePhone := []string{}
		for _, row := range ps {
			p1 := tools.IsEmpty(row["mobilephone"])
			if p1 == "" {
				continue
			}
			devicePhone = append(devicePhone, strings.Split(p1, ",")...)
		}
		if osname == "windows" {
			cpuuse_command := "wmic cpu get loadpercentage"
			memFree_command := "wmic OS get FreePhysicalMemory"
			memTotal_command := "wmic ComputerSystem get TotalPhysicalMemory"
			diskuse_command_c := "wmic LogicalDisk where Caption='C:' get FreeSpace,Size /value"
			cmd := exec.Command("cmd", "/C", cpuuse_command)
			info1, _ := cmd.Output()
			info := []byte{}
			for _, b := range info1 {
				if b == 32 || b == 13 || b == 10 {
					continue
				}
				info = append(info, b)
			}
			v := string(info)
			if v != "" {
				v = strings.Trim(strings.ReplaceAll(v, "LoadPercentage", ""), " ")
				vint, _ := strconv.Atoi(v)
				cpuHC_lst["ishc"] = vint
				if vint >= cpuHC {
					cpuHC_lst["hcnt"] = cpuHC_lst["hcnt"] + 1
					if cpuHC_lst["hcnt"] >= 3 {
						if !cpualarming {
							msgobj := map[string]interface{}{}
							msgobj["datatype"] = sms.TemplateCode_Device_Performance
							msgobj["params"] = []string{appname + thisIp, "CPU", fmt.Sprintf("已达%d", vint) + "%"}
							msgobj["phone"] = devicePhone
							DataMonitorMsgChan <- msgobj
							desc := fmt.Sprintf("[%s]系统CPU耗用告警:当前CPU耗用率已达%d%%,请尽快处理！同时系统将限制同时访问的SCD数量！", appname, vint)
							new(bo.SystemAlarm).Save("1", "cpu", desc, 0)
							if log_alarm_email != "" {
								new(email.Send).SendEmail(log_alarm_email, appname+"系统CPU耗用告警", desc)
							}
						}
						cpualarming = true
					}
					logger.Logger.Println(fmt.Sprintf("！！！警告：当前CPU耗用率已达%d,请尽快处理！同时系统将限制同时访问的SCD数量！", vint))
				} else {
					cpuHC_lst["hcnt"] = 0
					if cpualarming {
						desc := fmt.Sprintf("[%s]系统CPU耗用告警解除，当前CPU耗用率%d%%", appname, vint)
						new(bo.SystemAlarm).Save("1", "cpu", desc, 1)
						if log_alarm_email != "" {
							new(email.Send).SendEmail(log_alarm_email, appname+"系统CPU耗用告警解除", desc)
						}
					}
					cpualarming = false
				}
				global.PerformanceRuntimeMonitorResult.Store("cpu", vint)
			}
			//总内存.结果的单位为b
			cmd = exec.Command("cmd", "/C", memTotal_command)
			info1, _ = cmd.Output()
			info = []byte{}
			for _, b := range info1 {
				if b == 32 || b == 13 || b == 10 {
					continue
				}
				info = append(info, b)
			}
			v = string(info)
			totalmem := 0
			if v != "" {
				v = strings.Trim(strings.ReplaceAll(v, "TotalPhysicalMemory", ""), " ")
				totalmem, _ = strconv.Atoi(v)
			}
			//剩余内存.结果的单位为Kb
			cmd = exec.Command("cmd", "/C", memFree_command)
			info1, _ = cmd.Output()
			info = []byte{}
			for _, b := range info1 {
				if b == 32 || b == 13 || b == 10 {
					continue
				}
				info = append(info, b)
			}
			v = string(info)
			//logger.Logger.Debug(fmt.Sprintf("====总内存：%d 当前空闲内存：%s", totalmem, v))
			if v != "" {
				v = strings.Trim(strings.ReplaceAll(v, "FreePhysicalMemory", ""), " ")
				vint, _ := strconv.Atoi(v)
				if vint > 0 {
					memuserate := int((1 - (float32(vint)/1024/1024)/(float32(totalmem)/1024/1024/1024)) * 100)
					memHC_lst["ishc"] = memuserate
					if memuserate >= memHC {
						memHC_lst["hcnt"] = memHC_lst["hcnt"] + 1
						if memHC_lst["hcnt"] >= 3 {
							if !memalarming {
								msgobj := map[string]interface{}{}
								msgobj["datatype"] = sms.TemplateCode_Device_Performance
								msgobj["params"] = []string{appname + thisIp, "内存", fmt.Sprintf("已达%d", memuserate) + "%"}
								msgobj["phone"] = devicePhone
								DataMonitorMsgChan <- msgobj
								desc := fmt.Sprintf("[%s]系统内存耗用告警:当前内存耗用率已达%d%%,请尽快处理！同时系统将尝试清除访问频率较低的SCD缓存数据！", appname, memuserate)
								new(bo.SystemAlarm).Save("1", "mem", desc, 0)
								if log_alarm_email != "" {
									new(email.Send).SendEmail(log_alarm_email, appname+"系统内存耗用告警", desc)
								}
							}
							memalarming = true
							//释放SCD缓存
							go new(bo.ScdParse).RemoveScdCache()
						}
						logger.Logger.Println(fmt.Sprintf("！！！警告：当前内存耗用率已达%d,请尽快处理！同时系统将尝试清除访问频率较低的SCD缓存数据！", memuserate))

					} else {
						memHC_lst["hcnt"] = 0
						if memalarming {
							desc := fmt.Sprintf("[%s]系统内存耗用告警解除，当前内存使用率%d%%", appname, memuserate)
							new(bo.SystemAlarm).Save("1", "mem", desc, 1)
							if log_alarm_email != "" {
								new(email.Send).SendEmail(log_alarm_email, appname+"系统内存耗用告警解除", desc)
							}
						}
						memalarming = false
					}
					global.PerformanceRuntimeMonitorResult.Store("mem", memuserate)
				}
			}
			cmd = exec.Command("cmd", "/C", diskuse_command_c)
			info, _ = cmd.Output()
			v = string(info)
			freeSpace := 0
			totalSpace := 0
			//logger.Logger.Debug(fmt.Sprintf("====C盘信息：%+v", v))
			if v != "" {
				for _, tmpv := range strings.Split(strings.Trim(v, " "), "\r\n") {
					if strings.TrimSpace(tmpv) == "" {
						continue
					}
					tmpv = strings.ReplaceAll(tmpv, "\r", "")
					if strings.HasPrefix(tmpv, "FreeSpace") {
						freeSpace, err = strconv.Atoi(strings.ReplaceAll(tmpv, "FreeSpace=", ""))
						if err != nil {
							log.Println(err)
						}
					}
					if strings.HasPrefix(tmpv, "Size") {
						totalSpace, err = strconv.Atoi(strings.ReplaceAll(tmpv, "Size=", ""))
						if err != nil {
							log.Println(err)
						}
					}
				}
				//logger.Logger.Debug(fmt.Sprint("====total:%d  free:%d  ===alarm:%d", totalSpace, freeSpace, diskHCC))
				if freeSpace > 0 {
					diskHCC_lst["ishc"] = 100 - int(freeSpace*100/totalSpace)
					if diskHCC_lst["ishc"] >= diskHCC {
						diskHCC_lst["hcnt"] = diskHCC_lst["hcnt"] + 1
						if diskHCC_lst["hcnt"] >= 3 {
							if !diskcalarming {
								msgobj := map[string]interface{}{}
								msgobj["datatype"] = sms.TemplateCode_Device_Performance
								msgobj["params"] = []string{appname + thisIp, "系统C盘", fmt.Sprintf("已达%d", diskHCC_lst["ishc"]) + "%"}
								msgobj["phone"] = devicePhone
								DataMonitorMsgChan <- msgobj
								desc := fmt.Sprintf("[%s]磁盘C盘剩余空间告警:当前磁盘空间已使用%d%%,请尽快处理！", appname, diskHCC_lst["ishc"])
								new(bo.SystemAlarm).Save("1", "disk", desc, 0)
								if log_alarm_email != "" {
									new(email.Send).SendEmail(log_alarm_email, appname+"系统C盘剩余空间告警", desc)
								}
								diskcalarming = true
							}
						}
						logger.Logger.Println(fmt.Sprintf("！！！警告：当前系统磁盘占用率已达%d,请尽快处理！", diskHCC_lst["ishc"]))
					} else {
						diskHCC_lst["hcnt"] = 0
						if diskcalarming {
							desc := fmt.Sprintf("[%s]磁盘C盘剩余空间告警解除，当前使用空间%d%%", appname, diskHCC_lst["ishc"])
							new(bo.SystemAlarm).Save("1", "disk", desc, 1)
							if log_alarm_email != "" {
								new(email.Send).SendEmail(log_alarm_email, appname+"系统C盘剩余空间告警解除", desc)
							}
						}
						diskcalarming = false
					}
					global.PerformanceRuntimeMonitorResult.Store("disk_c", diskHCC_lst["ishc"])
				}
			}
			//当前执行路径
			exePath := os.Args[0]
			//fmt.Println("exePath:", exePath)
			exeDisk := strings.Split(exePath, "\\")[0]
			diskuse_command_n := "wmic LogicalDisk where Caption='" + exeDisk + "' get FreeSpace,Size /value"
			cmd = exec.Command("cmd", "/C", diskuse_command_n)
			info, _ = cmd.Output()
			v = string(info)
			if v != "" {
				for _, tmpv := range strings.Split(strings.Trim(v, " "), "\r\n") {
					if strings.TrimSpace(tmpv) == "" {
						continue
					}
					tmpv = strings.ReplaceAll(tmpv, "\r", "")
					if strings.HasPrefix(tmpv, "FreeSpace") {
						freeSpace, err = strconv.Atoi(strings.ReplaceAll(tmpv, "FreeSpace=", ""))
						if err != nil {
							log.Println(err)
						}
					}
					if strings.HasPrefix(tmpv, "Size") {
						totalSpace, err = strconv.Atoi(strings.ReplaceAll(tmpv, "Size=", ""))
						if err != nil {
							log.Println(err)
						}
					}
				}
				if freeSpace > 0 {
					diskHCI_lst["ishc"] = 100 - int(freeSpace*100/totalSpace)
					if diskHCI_lst["ishc"] >= diskHCI {
						diskHCI_lst["hcnt"] = diskHCI_lst["hcnt"] + 1
						if diskHCI_lst["hcnt"] >= 3 {
							if !diskalarming {
								msgobj := map[string]interface{}{}
								msgobj["datatype"] = sms.TemplateCode_Device_Performance
								msgobj["params"] = []string{appname + thisIp, "应用" + exeDisk[0:1] + "盘", fmt.Sprintf("已达%d", diskHCI_lst["ishc"]) + "%"}
								msgobj["phone"] = devicePhone
								DataMonitorMsgChan <- msgobj
								desc := fmt.Sprintf("[%s]磁盘%s盘剩余空间告警:当前磁盘空间已使用%d%%,请尽快处理！", appname, exeDisk[0:1], diskHCI_lst["ishc"])
								new(bo.SystemAlarm).Save("1", "disk", desc, 0)
								if log_alarm_email != "" {
									new(email.Send).SendEmail(log_alarm_email, appname+"系统"+exeDisk[0:1]+"盘剩余空间告警", desc)
								}
								diskalarming = true
							}
							logger.Logger.Println(fmt.Sprintf("！！！警告：当前磁盘%s占用率已达%d,请尽快处理！", exeDisk[0:1], diskHCI_lst["ishc"]))
						}
					} else {
						diskHCI_lst["hcnt"] = 0
						if diskalarming {
							desc := fmt.Sprintf("[%s]磁盘%s盘剩余空间告警已解除，当前磁盘使用%d%%", appname, exeDisk[0:1], diskHCI_lst["ishc"])
							new(bo.SystemAlarm).Save("1", "disk", desc, 1)
							if log_alarm_email != "" {
								new(email.Send).SendEmail(log_alarm_email, appname+"系统"+exeDisk[0:1]+"盘剩余空间告警解除", desc)
							}
						}
						diskalarming = false
					}
					global.PerformanceRuntimeMonitorResult.Store("disk", diskHCC_lst["ishc"])
				}
			}
		} else {
			//linux系统
			v, errmem := mem.VirtualMemory()
			if errmem != nil {
				logger.Logger.Error(errmem)
			} else {
				//fmt.Printf("内存使用率%d\n", v.UsedPercent)
				memuserate := int(v.UsedPercent)
				memHC_lst["ishc"] = memuserate
				if memuserate >= memHC {
					memHC_lst["hcnt"] = memHC_lst["hcnt"] + 1
					if memHC_lst["hcnt"] >= 3 {
						if !memalarming {
							msgobj := map[string]interface{}{}
							msgobj["datatype"] = sms.TemplateCode_Device_Performance
							msgobj["params"] = []string{appname + thisIp, "内存", fmt.Sprintf("已达%d", memuserate) + "%"}
							msgobj["phone"] = devicePhone
							DataMonitorMsgChan <- msgobj
							desc := fmt.Sprintf("[%s]系统内存耗用告警:当前内存已耗用%d%%,请尽快处理！同时系统将尝试清除访问频率较低的SCD缓存数据！", appname, memuserate)
							new(bo.SystemAlarm).Save("1", "mem", desc, 0)
							if log_alarm_email != "" {
								new(email.Send).SendEmail(log_alarm_email, appname+"系统内存耗用告警", desc)
							}
						}
						memalarming = true
						//释放SCD缓存
						go new(bo.ScdParse).RemoveScdCache()
					}
					logger.Logger.Println(fmt.Sprintf("！！！警告：当前内存耗用率已达%d,请尽快处理！同时系统将尝试清除访问频率较低的SCD缓存数据！", memuserate))
				} else {
					memHC_lst["hcnt"] = 0
					if memalarming {
						desc := fmt.Sprintf("[%s]系统内存耗用告警已解除，当前内存耗用%d%%", appname, memuserate)
						new(bo.SystemAlarm).Save("1", "mem", desc, 1)
						if log_alarm_email != "" {
							new(email.Send).SendEmail(log_alarm_email, appname+"系统内存耗用告警解除", desc)
						}
					}
					memalarming = false
				}
				global.PerformanceRuntimeMonitorResult.Store("mem", memuserate)
			}
			// CPU使用率
			//for {
			percent, errcpu := cpu.Percent(time.Second, true)
			if errcpu != nil {
				logger.Logger.Error(errcpu)
			} else {
				//fmt.Printf("cpu percent:%v", percent)
				cpuT := 0.0
				for _, v := range percent {
					cpuT = cpuT + v
				}
				cpuavg := int(cpuT / float64(len(percent)))
				cpuHC_lst["ishc"] = cpuavg
				//fmt.Printf("CPU使用率%d\n", cpuavg)
				if cpuavg >= cpuHC {
					cpuHC_lst["hcnt"] = cpuHC_lst["hcnt"] + 1
					if cpuHC_lst["hcnt"] >= 3 {
						if !cpualarming {
							msgobj := map[string]interface{}{}
							msgobj["datatype"] = sms.TemplateCode_Device_Performance
							msgobj["params"] = []string{appname + thisIp, "CPU", fmt.Sprintf("已达%d", cpuavg) + "%"}
							msgobj["phone"] = devicePhone
							DataMonitorMsgChan <- msgobj
							desc := fmt.Sprintf("[%s]系统CPU耗用告警:当前CPU已耗用%d%%,请尽快处理！同时系统将限制同时访问的SCD数量！", appname, cpuavg)
							new(bo.SystemAlarm).Save("1", "cpu", desc, 0)
							if log_alarm_email != "" {
								new(email.Send).SendEmail(log_alarm_email, appname+"系统CPU耗用告警", desc)
							}
						}
						cpualarming = true
					}
					logger.Logger.Println(fmt.Sprintf("！！！警告：当前CPU耗用率已达%d,请尽快处理！同时系统将限制同时访问的SCD数量！", cpuavg))
				} else {
					cpuHC_lst["hcnt"] = 0
					if cpualarming {
						desc := fmt.Sprintf("[%s]系统CPU耗用告警已解除", appname, cpuavg)
						new(bo.SystemAlarm).Save("1", "cpu", desc, 1)
						if log_alarm_email != "" {
							new(email.Send).SendEmail(log_alarm_email, appname+"系统CPU耗用告警解除", desc)
						}
					}
					cpualarming = false
				}
				global.PerformanceRuntimeMonitorResult.Store("cpu", cpuavg)
			}
			parts, err2 := disk.Partitions(true)
			if err2 != nil {
				logger.Logger.Error(err2)
			} else {
				diskT := 0.0
				diskC := 0
				for _, part := range parts {
					if !strings.HasPrefix(part.Device, "/dev") {
						continue
					}
					//fmt.Printf("part:%v", part.String())
					diskInfo, _ := disk.Usage(part.Mountpoint)
					diskT = diskT + diskInfo.UsedPercent
					diskC = diskC + 1
					//fmt.Printf("disk info:used:%v free:%v", diskInfo.UsedPercent, diskInfo.Free)
				}
				avg := int(diskT / float64(diskC))
				diskHCC_lst["ishc"] = avg
				//fmt.Printf("磁盘使用率%d\n", avg)
				if diskHCC_lst["ishc"] >= diskHCC {
					diskHCC_lst["hcnt"] = diskHCC_lst["hcnt"] + 1
					if diskHCC_lst["hcnt"] >= 3 {
						if !diskalarming {
							msgobj := map[string]interface{}{}
							msgobj["datatype"] = sms.TemplateCode_Device_Performance
							msgobj["params"] = []string{appname + thisIp, "数据磁盘", fmt.Sprintf("已达%d", avg) + "%"}
							msgobj["phone"] = devicePhone
							DataMonitorMsgChan <- msgobj
							desc := fmt.Sprintf("[%s]磁盘空间告警:当前磁盘空间已使用%d%%,请尽快释放多余的空间！", appname, avg)
							new(bo.SystemAlarm).Save("1", "disk", desc, 0)
							if log_alarm_email != "" {
								new(email.Send).SendEmail(log_alarm_email, appname+"系统CPU耗用告警", desc)
							}
						}
						diskalarming = true
					}
					logger.Logger.Println(fmt.Sprintf("！！！警告：当前磁盘空间占用率已达%d,请尽快释放多余的空间！", avg))
				} else {
					diskHCC_lst["hcnt"] = 0
					if diskalarming {
						desc := fmt.Sprintf("[%s]磁盘空间告警已解除", appname, avg)
						new(bo.SystemAlarm).Save("1", "disk", desc, 1)
						if log_alarm_email != "" {
							new(email.Send).SendEmail(log_alarm_email, appname+"磁盘空间告警解除", desc)
						}
					}
					diskalarming = false
				}
				global.PerformanceRuntimeMonitorResult.Store("disk", avg)
			}
		}
		time.Sleep(1 * time.Minute)
	}
}

//启动实时性能监控。每5秒一次并返回结果
func (t *DataMonitor) PerformanceRuntimeMonitorStart() {
	if t.performanceRuntimeMonitorState {
		return
	}
	t.pmResultLock = sync.RWMutex{}
	t.performanceRuntimeMonitorResult = map[string]interface{}{}
	var err error
	osname := string(runtime.GOOS)
	cpuHC := 80   //cpu高耗用率
	memHC := 80   //内存高耗用率
	diskHCC := 90 //C磁盘高耗用率
	diskHCI := 90 //程序所在磁盘高耗用率
	cpuHC_lst := map[string]int{"ishc": 0, "hcnt": 0, "lcnt": 0}
	memHC_lst := map[string]int{"ishc": 0, "hcnt": 0, "lcnt": 0}
	diskHCC_lst := map[string]int{"ishc": 0, "hcnt": 0, "lcnt": 0}
	diskHCI_lst := map[string]int{"ishc": 0, "hcnt": 0, "lcnt": 0}
	t.performanceRuntimeMonitorState = true
	for {
		if !t.performanceRuntimeMonitorState {
			break
		}
		t.pmResultLock.Lock()
		if osname == "windows" {
			cpuuse_command := "wmic cpu get loadpercentage"
			memFree_command := "wmic OS get FreePhysicalMemory"
			memTotal_command := "wmic ComputerSystem get TotalPhysicalMemory"
			diskuse_command_c := "wmic LogicalDisk where Caption='C:' get FreeSpace,Size /value"
			cmd := exec.Command("cmd", "/C", cpuuse_command)
			info1, _ := cmd.Output()
			info := []byte{}
			for _, b := range info1 {
				if b == 32 || b == 13 || b == 10 {
					continue
				}
				info = append(info, b)
			}
			v := string(info)
			//log.Println(fmt.Sprintf("====CPU：%s", v))
			if v != "" {
				v = strings.Trim(strings.ReplaceAll(v, "LoadPercentage", ""), " ")
				vint, _ := strconv.Atoi(v)
				cpuHC_lst["ishc"] = vint
				if vint >= cpuHC {
					cpuHC_lst["hcnt"] = cpuHC_lst["hcnt"] + 1
					if cpuHC_lst["hcnt"] >= 3 {
						t.performanceRuntimeMonitorResult["cpu"] = cpuHC_lst
						log.Println(fmt.Sprintf("====CPU：%s", v))
					}
				} else {
					cpuHC_lst["hcnt"] = 0
				}
			}
			//总内存
			cmd = exec.Command("cmd", "/C", memTotal_command)
			info1, _ = cmd.Output()
			info = []byte{}
			for _, b := range info1 {
				if b == 32 || b == 13 || b == 10 {
					continue
				}
				info = append(info, b)
			}
			v = string(info)
			totalmem := int64(0)
			if v != "" {
				v = strings.Trim(strings.ReplaceAll(v, "TotalPhysicalMemory", ""), " ")
				totalmem, _ = strconv.ParseInt(v, 10, 64)
			}
			cmd = exec.Command("cmd", "/C", memFree_command)
			info1, _ = cmd.Output()
			info = []byte{}
			for _, b := range info1 {
				if b == 32 || b == 13 || b == 10 {
					continue
				}
				info = append(info, b)
			}
			v = string(info)
			//log.Println(fmt.Sprintf("====总内存%d free：%s", totalmem, v))
			if v != "" {
				v = strings.Trim(strings.ReplaceAll(v, "FreePhysicalMemory", ""), " ")
				vint, _ := strconv.ParseInt(v, 10, 64)
				if vint > 0 {
					var useMem = totalmem - vint*1024
					memuserate := (useMem * 100 / totalmem)
					//log.Println(fmt.Sprintf("====内存占用率：%d/%d %d", useMem, totalmem, memuserate))
					memHC_lst["ishc"] = int(memuserate)
					if int(memuserate) >= memHC {
						memHC_lst["hcnt"] = memHC_lst["hcnt"] + 1
						if memHC_lst["hcnt"] >= 3 {
							t.performanceRuntimeMonitorResult["mem"] = memHC_lst
							log.Println(fmt.Sprintf("====内存占用率：%d/%d %d", useMem, totalmem, memuserate))
						}
					} else {
						memHC_lst["hcnt"] = 0
					}
				}
			}
			cmd = exec.Command("cmd", "/C", diskuse_command_c)
			info, _ = cmd.Output()
			v = string(info)
			freeSpace := 0
			totalSpace := 0
			if v != "" {
				for _, tmpv := range strings.Split(strings.Trim(v, " "), "\r\n") {
					if strings.TrimSpace(tmpv) == "" {
						continue
					}
					tmpv = strings.ReplaceAll(tmpv, "\r", "")
					if strings.HasPrefix(tmpv, "FreeSpace") {
						freeSpace, err = strconv.Atoi(strings.ReplaceAll(tmpv, "FreeSpace=", ""))
						if err != nil {
							log.Println(err)
						}
					}
					if strings.HasPrefix(tmpv, "Size") {
						totalSpace, err = strconv.Atoi(strings.ReplaceAll(tmpv, "Size=", ""))
						if err != nil {
							log.Println(err)
						}
					}
				}
				if freeSpace > 0 {
					diskHCC_lst["ishc"] = 100 - int(freeSpace*100/totalSpace)
					if diskHCC_lst["ishc"] >= diskHCC {
						diskHCC_lst["hcnt"] = diskHCC_lst["hcnt"] + 1
						if diskHCC_lst["hcnt"] >= 3 {
							t.performanceRuntimeMonitorResult["disk"] = diskHCC_lst
						}
					} else {
						diskHCC_lst["hcnt"] = 0
					}
				}
			}
			//当前执行路径
			exePath := os.Args[0]
			//fmt.Println("exePath:", exePath)
			exeDisk := strings.Split(exePath, "\\")[0]
			diskuse_command_n := "wmic LogicalDisk where Caption='" + exeDisk + "' get FreeSpace,Size /value"
			cmd = exec.Command("cmd", "/C", diskuse_command_n)
			info, _ = cmd.Output()
			v = string(info)
			if v != "" {
				for _, tmpv := range strings.Split(strings.Trim(v, " "), "\r\n") {
					if strings.TrimSpace(tmpv) == "" {
						continue
					}
					tmpv = strings.ReplaceAll(tmpv, "\r", "")
					if strings.HasPrefix(tmpv, "FreeSpace") {
						freeSpace, err = strconv.Atoi(strings.ReplaceAll(tmpv, "FreeSpace=", ""))
						if err != nil {
							log.Println(err)
						}
					}
					if strings.HasPrefix(tmpv, "Size") {
						totalSpace, err = strconv.Atoi(strings.ReplaceAll(tmpv, "Size=", ""))
						if err != nil {
							log.Println(err)
						}
					}
				}
				if freeSpace > 0 {
					diskHCI_lst["ishc"] = 100 - int(freeSpace*100/totalSpace)
					if diskHCI_lst["ishc"] >= diskHCI {
						diskHCI_lst["hcnt"] = diskHCI_lst["hcnt"] + 1
						if diskHCI_lst["hcnt"] >= 3 {
							t.performanceRuntimeMonitorResult["disk_"+exeDisk] = diskHCC_lst
						}
					} else {
						diskHCI_lst["hcnt"] = 0
					}
				}
			}
		} else {
			//linux系统
			v, errmem := mem.VirtualMemory()
			if errmem != nil {
				log.Println(errmem)
			} else {
				//log.Printf("内存使用率%d\n", v.UsedPercent)
				memuserate := int(v.UsedPercent)
				memHC_lst["ishc"] = memuserate
				if memuserate >= memHC {
					memHC_lst["hcnt"] = memHC_lst["hcnt"] + 1
					if memHC_lst["hcnt"] >= 3 {
						t.performanceRuntimeMonitorResult["mem"] = memHC_lst
						log.Printf("内存使用率%d\n", v.UsedPercent)
					}
				} else {
					memHC_lst["hcnt"] = 0
				}
			}
			percent, errcpu := cpu.Percent(time.Second, true)
			if errcpu != nil {
				log.Println(errcpu)
			} else {
				//fmt.Printf("cpu percent:%v", percent)
				cpuT := 0.0
				for _, v := range percent {
					cpuT = cpuT + v
				}
				cpuavg := int(cpuT / float64(len(percent)))
				cpuHC_lst["ishc"] = cpuavg
				//fmt.Printf("CPU使用率%d\n", cpuavg)
				if cpuavg >= cpuHC {
					cpuHC_lst["hcnt"] = cpuHC_lst["hcnt"] + 1
					if cpuHC_lst["hcnt"] >= 3 {
						t.performanceRuntimeMonitorResult["cpu"] = cpuHC_lst
						log.Printf("CPU使用率%d\n", cpuavg)
					}
				} else {
					cpuHC_lst["hcnt"] = 0
				}
			}
			parts, err2 := disk.Partitions(true)
			if err2 != nil {
				log.Println("get Partitions failed, err:%v", err2)
			} else {
				diskT := 0.0
				diskC := 0
				for _, part := range parts {
					if !strings.HasPrefix(part.Device, "/dev") {
						continue
					}
					//fmt.Printf("part:%v", part.String())
					diskInfo, _ := disk.Usage(part.Mountpoint)
					diskT = diskT + diskInfo.UsedPercent
					diskC = diskC + 1
					//fmt.Printf("disk info:used:%v free:%v", diskInfo.UsedPercent, diskInfo.Free)
				}
				avg := int(diskT / float64(diskC))
				diskHCC_lst["ishc"] = avg
				//fmt.Printf("磁盘使用率%d\n", avg)
				if diskHCC_lst["ishc"] >= diskHCC {
					diskHCC_lst["hcnt"] = diskHCC_lst["hcnt"] + 1
					if diskHCC_lst["hcnt"] >= 3 {
						t.performanceRuntimeMonitorResult["disk"] = diskHCC_lst
						log.Printf("磁盘使用率%d\n", avg)
					}
				} else {
					diskHCC_lst["hcnt"] = 0
				}
			}
		}
		t.pmResultLock.Unlock()
		//log.Println(fmt.Sprintf("====实时性能监控结果:%v", t.performanceRuntimeMonitorResult))
		time.Sleep(5 * time.Second)
	}
}

//停止实时性能监控
func (t *DataMonitor) PerformanceRuntimeMonitorStop() {
	t.performanceRuntimeMonitorState = false
}

//过期附件监测
func attachmentMonitorStart() {
	log.Println("启动过期附件自动清理进程")
	for {
		nowHour := time.Now().Format("15")
		//每天早上0点清理一次
		dataTimoutMonitorHour, _ := bo.GetSysParamValue("attachmentClearHour", "00")
		if nowHour != dataTimoutMonitorHour {
			time.Sleep(time.Minute * 50)
			continue
		}
		m := new(DataMonitor)
		m.attachmentAutoClear()
		time.Sleep(time.Minute * 50)
	}
}

func (t *DataMonitor) OptimizeTable() {
	db := orm.NewOrm()
	//获取scd属性表
	sql := "select table_name , table_comment from information_schema.tables where table_schema = (select database()) and table_name like 't_%' "
	rowset := []orm.Params{}
	_, err := db.Raw(sql).Values(&rowset)
	if err != nil {
		log.Println(err)
		return
	}
	for _, row := range rowset {
		tb := tools.IsEmpty(row["TABLE_NAME"]) + tools.IsEmpty(row["table_name"])
		_, err = db.Raw("optimize table " + tb).Exec()
		if err != nil {
			log.Println(err)
		}
		_, err = db.Raw("analyze table " + tb).Exec()
		if err != nil {
			log.Println(err)
		}
	}
}

//临时scd数据清理
func (t *DataMonitor) tempScdClear() {
	db := orm.NewOrm()
	t1 := time.Now().Unix()
	clearT := t1 - int64(1*60*60) //1小时之前
	clearDate := time.Unix(clearT, 0).Format("2006-01-02 15:04:05")
	sql := "select * from t_scd_scl where id not in(select scd_id from t_data_task) and created_time<?"
	rowset := []orm.Params{}
	_, err := db.Raw(sql, clearDate).Values(&rowset)
	if err != nil {
		log.Println(err)
	} else {
		if len(rowset) == 0 {
			return
		}
		uinfo := map[string]interface{}{}
		uinfo["name"] = "临时SCD清理任务"
		for _, row := range rowset {
			scd := new(bo.ScdMgr)
			scd.SetUserInfo(uinfo)
			scd.DeleteScd(tools.IsEmpty(row["id"]), true)
		}
	}
}

//附件自动清理
func (t *DataMonitor) attachmentAutoClear() {
	db := orm.NewOrm()
	sql := "select * from t_data_attachment where skeepday>0"
	list := []orm.Params{}
	_, err := db.Raw(sql).Values(&list)
	if err != nil {
		log.Println(err)
		return
	}
	if len(list) == 0 {
		return
	}
	for _, row := range list {
		skeepday := tools.IsEmpty(row["skeepday"])
		colname := tools.IsEmpty(row["colname"])
		tname := tools.IsEmpty(row["tablename"])
		sql1 := "select id," + colname + " from " + tname + " where " + colname + "!='' and CREATED_TIME<DATE_ADD(now(),INTERVAL -" + skeepday + " DAY)"
		where := tools.IsEmpty(row["filterwhere"])
		if where != "" {
			sql1 = sql1 + " and " + where
		}
		sql1 = sql1 + " limit 0,500 "
		for {
			tmpresultset := []orm.Params{}
			_, err = db.Raw(sql1).Values(&tmpresultset)
			if err != nil {
				log.Println(err)
				break
			}
			if len(tmpresultset) == 0 {
				break
			}
			ids := []string{}
			for _, r := range tmpresultset {
				ids = append(ids, tools.IsEmpty(r["id"]))
				imgsrc := tools.IsEmpty(r[colname])
				if strings.Index(imgsrc, "static/download/") > -1 {
					reg, _ := regexp.Compile("http.*?static/download/")
					imgsrc = reg.ReplaceAllString(imgsrc, "static/download/")
					imgsrc = "./" + imgsrc //物理存储路径
					//删除文件
					os.Remove(imgsrc)
				}
			}
			_, err = db.Raw("update " + tname + " set " + colname + "='' where id in('" + strings.Join(ids, "','") + "')").Exec()
			if err != nil {
				log.Println(err)
				break
			}
		}
	}
}

func (t *DataMonitor) sendDeviceSms(devicename string, devicelist []string, datatype string, timeout int, devicePhone string) {
	if len(devicelist) > 0 {
		h := float64(timeout / 3600)
		cnt := len(devicelist)
		//最多显示5个设备名称
		if cnt > 5 {
			cnt = 5
		}
		devicelistNames := strings.Join(devicelist[:cnt], ",")

		msgobj := map[string]interface{}{}
		msgobj["datatype"] = sms.TemplateCode_Device_Offline
		msgobj["params"] = []string{devicename, fmt.Sprintf("%s(共%d台)", devicelistNames, cnt), fmt.Sprintf("%d小时", int(math.Floor(h)))}
		msgobj["phone"] = strings.Split(devicePhone, ",")
		DataMonitorMsgChan <- msgobj
	}
}