ZLPMS/PMS.BusinessService/CloudMonitorManage/CloudMonitorBusiness.cs

using PMS.BusinessModels.CloudMonitorManage;
using PMS.BusinessModels.SMS;
using PMS.BusinessService.SysManager;
using PMS.DBService.CloudMonitorManage;
using PMS.DBService.SysManager;
using PMS.Interface;
using PMS.Interface.CloudMonitorManage;
using PMS.Interface.MessageManage;
using PMS.Plugins.Common;
using QWPlatform.IService;
using QWPlatform.SystemLibrary;
using QWPlatform.SystemLibrary.LogManager;
using QWPlatform.SystemLibrary.Utils;
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Data;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using static System.Collections.Specialized.BitVector32;
using System.Web.Caching;

namespace PMS.BusinessService.CloudMonitorManage
{
    public class CloudMonitorBusiness : ICloudMonitor
    {
        private ConfigDBService configDBService = DataServiceBase.Instance<ConfigDBService>();
        private CloudMonitorService dbCloudMonitor = DataServiceBase.Instance<CloudMonitorService>();
        //系统配置
        SysConfigBLLServier configBll = QWPlatform.SystemLibrary.Provider.SingletonProvider<SysConfigBLLServier>.Instance;
        //推送消息
        IMessageManage notice = InterfaceFactory.CreateBusinessInstance<IMessageManage>();
        //发布地址
        public static string publishingAddress = ConfigurationManager.AppSettings["PublishingAddress"].ToStringEx();
        private static Logger Log = Logger.Instance;

        /// <summary>
        /// 云监控配置
        /// </summary>
        private static CloudMonitorConfig _cloudMonitorConfig = null;

        /// <summary>
        /// key:服务器id value:服务器指标id
        /// </summary>
        private static Dictionary<string, string> _cacheIndexId = new Dictionary<string, string>();

        /// <summary>
        /// 持续预警
        /// </summary>
        private static Dictionary<string, object> _cacheDuration = new Dictionary<string, object>();

        /// <summary>
        /// 自定义预警数据
        /// </summary>
        private static List<CustomAlertModel> _cacheCustomAlertList;

        /// <summary>
        /// 项目监控数据
        /// </summary>
        private static List<ProjectMonitorModel> _cacheProjectMonitorList;

        #region 云监控配置相关
        /// <summary>
        /// 保存总配置
        /// </summary>
        /// <param name="cloudMonitorConfig"></param>
        /// <returns></returns>
        public bool SaveCloudMonitorConfig(CloudMonitorConfig cloudMonitorConfig)
        {
            //如果有保存则清空缓存
            _cloudMonitorConfig = null;
            return configDBService.SaveCloudMonitorConfig(cloudMonitorConfig);
        }

        /// <summary>
        /// 保存采集配置
        /// </summary>
        /// <param name="collectionConfig"></param>
        /// <returns></returns>
        public bool SaveCollectionConfig(CollectionConfig collectionConfig)
        {
            var config = GetCloudMonitorConfig();
            config.CollectionConfig = collectionConfig;
            WebUI.Jobs.SchedulerMain.CollectionInterruptJob(collectionConfig.采集服务中断执行频率);
            return SaveCloudMonitorConfig(config);
        }

        /// <summary>
        /// 保存预警配置
        /// </summary>
        /// <param name="alertConfig"></param>
        /// <returns></returns>
        public bool SaveAlertConfig(AlertConfig alertConfig)
        {
            var config = GetCloudMonitorConfig();
            config.AlertConfig = alertConfig;
            return SaveCloudMonitorConfig(config);
        }

        /// <summary>
        /// 获取配置
        /// </summary>
        /// <returns></returns>
        public CloudMonitorConfig GetCloudMonitorConfig()
        {
            if (_cloudMonitorConfig == null)
            {//如果有缓存则获取缓存的信息
                _cloudMonitorConfig = configDBService.GetCloudMonitorConfig();
            }
            return _cloudMonitorConfig;
        }

        #endregion

        #region 自定义预警配置相关
        /// <summary>
        /// 根据项目id获取自定义配置
        /// </summary>
        /// <param name="itemId"></param>
        /// <returns></returns>
        public List<CustomAlertModel> GetCustomAlertModelByItemId(string itemId)
        {
            var data = dbCloudMonitor.GetCustomAlertModelByItemId(itemId);
            return data;
        }

        /// <summary>
        /// 根据项目id获取 启动的自定义预警配置
        /// </summary>
        /// <param name="itemId"></param>
        /// <returns></returns>
        public List<CustomAlertModel> GetCustomAlertModelByItemIdAndStart(string itemId)
        {
            var data = dbCloudMonitor.GetCustomAlertModelByItemIdAndStart(itemId);
            return data;
        }

        /// <summary>
        /// 根据项目id获取 启动的自定义服务配置
        /// </summary>
        /// <param name="itemId"></param>
        /// <returns></returns>
        public List<CustomAPIModel> GetCustomServerModelByItemIdAndStart(string itemId)
        {
            var data = dbCloudMonitor.GetCustomServerModelByItemIdAndStart(itemId);
            return data;
        }

        /// <summary>
        /// 根据渠道id获取自定义配置
        /// </summary>
        /// <param name="channelId"></param>
        /// <returns></returns>
        public List<CustomAlertModel> GetCustomAlertModelByChannelId(string channelId, string search)
        {
            var data = dbCloudMonitor.GetCustomAlertModelByChannelId(channelId, search);
            return data;
        }

        /// <summary>
        /// 根据渠道id获取自定义API配置
        /// </summary>
        /// <param name="channelId"></param>
        /// <returns></returns>
        public List<CustomAPIModel> GetCustomAPIModelByChannelId(string channelId, string search)
        {
            var data = dbCloudMonitor.GetCustomAPIModelByChannelId(channelId, search);
            return data;
        }

        /// <summary>
        /// 根据创建人id获取自定义配置
        /// </summary>
        /// <param name="creatorId"></param>
        /// <returns></returns>
        public List<CustomAlertModel> GetCustomAlertModelByCreatorId(string creatorId)
        {
            var data = dbCloudMonitor.GetCustomAlertModelByCreatorId(creatorId);
            return data;
        }

        /// <summary>
        /// 新增自定义配置
        /// </summary>
        /// <param name="model"></param>
        /// <returns></returns>
        public bool InsertCustomAlert(CustomAlertModel model)
        {
            _cacheCustomAlertList = null;
            var data = dbCloudMonitor.InsertCustomAlert(model);
            return data;
        }

        /// <summary>
        /// 新增自定义API配置
        /// </summary>
        /// <param name="model"></param>
        /// <returns></returns>
        public bool InsertCustomAPI(CustomAPIModel model)
        {
            _cacheCustomAlertList = null;
            var data = dbCloudMonitor.InsertCustomAPI(model);
            return data;
        }

        /// <summary>
        /// 修改
        /// </summary>
        /// <param name="model"></param>
        /// <returns></returns>
        public bool UpdateCustomAlert(CustomAlertModel model)
        {
            _cacheCustomAlertList = null;
            var data = dbCloudMonitor.UpdateCustomAlert(model);
            return data;
        }

        /// <summary>
        /// 修改
        /// </summary>
        /// <param name="model"></param>
        /// <returns></returns>
        public bool UpdateCustomAPI(CustomAPIModel model)
        {
            _cacheCustomAlertList = null;
            var data = dbCloudMonitor.UpdateCustomAPI(model);
            return data;
        }

        /// <summary>
        /// 根据id删除
        /// </summary>
        /// <param name="id"></param>
        /// <returns></returns>
        public bool DeleteCustomAlertById(string id)
        {
            _cacheCustomAlertList = null;
            var data = dbCloudMonitor.DeleteCustomAlertById(id);
            return data;
        }

        /// <summary>
        /// 根据id删除
        /// </summary>
        /// <param name="id"></param>
        /// <returns></returns>
        public bool DeleteCustomAPIById(string id)
        {
            var data = dbCloudMonitor.DeleteCustomAPIById(id);
            return data;
        }

        /// <summary>
        /// 获取启动的所有自定义预警配置
        /// </summary>
        /// <returns></returns>
        public List<CustomAlertModel> GetCustomAlertModelByStart()
        {
            if (_cacheCustomAlertList == null)
                _cacheCustomAlertList = dbCloudMonitor.GetCustomAlertModelByStart();
            return _cacheCustomAlertList;
        }

        #endregion

        #region 服务器指标信息相关

        /// <summary>
        /// 根据项目id获取服务器信息
        /// </summary>
        /// <param name="itemid"></param>
        /// <returns></returns>
        public List<ServerInfoModel> GetServerByItemId(string itemid)
        {
            var data = dbCloudMonitor.GetServerByItemId(itemid);
            var serverIds = string.Join(",", data.Select(m => m.ID).ToList());
            if (!string.IsNullOrEmpty(serverIds))
            {
                var dbs = GetDbInfoByServerId(serverIds);
                data.ForEach(m =>
                {
                    m.数据库 = dbs.Where(n => m.ID == n.服务器ID).ToList();
                });
            }
            return data;
        }

        /// <summary>
        /// 获取所有服务器指标信息
        /// </summary>
        /// <returns></returns>
        public List<ServerIndexInfoModel> GetServerIndexInfoAll()
        {
            var data = dbCloudMonitor.GetServerIndexInfoAll();
            WorkWithColor(data);
            return data;
        }

        /// <summary>
        /// 获取所有服务器指标信息 通过项目id过滤
        /// </summary>
        /// <returns></returns>
        public List<ServerIndexInfoModel> GetServerIndexInfoByProjectId(string itemId)
        {
            var data = dbCloudMonitor.GetServerIndexInfoByProjectId(itemId);
            WorkWithColor(data);
            return data;
        }

        /// <summary>
        /// 获取所有服务器指标信息 通过渠道id过滤
        /// </summary>
        /// <returns></returns>
        public List<ServerIndexInfoModel> GetServerIndexInfoByChannelId(string channelId, string search = "")
        {
            var data = dbCloudMonitor.GetServerIndexInfoByChannelId(channelId, search);
            WorkWithColor(data);
            return data;
        }

        /// <summary>
        /// 获取指标详情 根据指标id
        /// </summary>
        /// <param name="indexId"></param>
        public List<ServerIndexDetailModel> GetServerIndexDetailByIndexId(string indexId, DateTime startTime, DateTime endTime)
        {
            var data = dbCloudMonitor.GetServerIndexDetailByIndexId(indexId, startTime, endTime);
            return data;
        }

        /// <summary>
        /// 获取指标详情 根据服务器id
        /// </summary>
        /// <param name="serverId"></param>
        public List<ServerIndexDetailModel> GetServerIndexDetailByServerId(string serverId, DateTime startTime, DateTime endTime)
        {
            var data = dbCloudMonitor.GetServerIndexDetailByServerId(serverId, startTime, endTime);
            return data;
        }

        /// <summary>
        /// 处理小数精度和颜色
        /// </summary>
        /// <param name="models"></param>
        private void WorkWithColor(List<ServerIndexInfoModel> models)
        {
            var config = GetCloudMonitorConfig().AlertConfig;
            foreach (var item in models)
            {
                var currentValue = Math.Round(item.CPU使用率, 2);
                item.CPU使用率 = currentValue;
                if (config.CPU使用濒危 <= currentValue && currentValue < config.CPU使用高危)
                {
                    item.CPU状态 = "濒危";
                }
                else if (config.CPU使用高危 <= currentValue)
                {
                    item.CPU状态 = "高危";
                }

                currentValue = Math.Round(item.内存使用率, 2);
                item.内存使用率 = currentValue;
                if (config.内存使用濒危 <= currentValue && currentValue < config.内存使用高危)
                {
                    item.内存状态 = "濒危";
                }
                else if (config.内存使用高危 <= currentValue)
                {
                    item.内存状态 = "高危";
                }

                currentValue = Math.Round(item.磁盘使用率, 2);
                item.磁盘使用率 = currentValue;
                if (config.磁盘使用濒危 <= currentValue && currentValue < config.磁盘使用高危)
                {
                    item.磁盘状态 = "濒危";
                }
                else if (config.磁盘使用高危 <= currentValue)
                {
                    item.磁盘状态 = "高危";
                }
            }
        }

        #endregion

        #region 服务器预警相关

        /// <summary>
        /// 插入服务器预警
        /// </summary>
        /// <param name="model"></param>
        /// <returns></returns>
        public string InsertServerAlert(ServerAlertDetailModel model)
        {
            if (string.IsNullOrEmpty(model.外部唯一键))
            {
                model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
            }
            var serverAlert = dbCloudMonitor.GetServerAlertByServerIdAndAlertType(model.外部唯一键);
            //没查询到，代表当前没得同类型预警，插入预警
            if (serverAlert == null)
            {
                ServerAlertModel alertModel = new ServerAlertModel
                {
                    ID = Guid.NewGuid().ToString("D"),
                    服务器ID = model.服务器ID,
                    预警类型 = model.预警类型,
                    创建时间 = model.最后修改时间,
                    项目ID = model.项目ID,
                    最后上报时间 = DateTime.Now,
                    外部唯一键 = model.外部唯一键
                };
                dbCloudMonitor.InsertServerAlert(alertModel);
                model.预警ID = alertModel.ID;
                dbCloudMonitor.InsertServerAlertDetail(model);
                //通知线程（邮件等）
                ThreadPool.QueueUserWorkItem(new WaitCallback(SendNotice), alertModel.ID);
            }
            else
            {
                model.预警ID = serverAlert.ID;
                var detailModel = dbCloudMonitor.GetServerAlertDetailByAlertIdAndAlertNameAndAlertLevel(model.预警ID, model.预警名称, model.预警等级);
                if (detailModel == null)
                {
                    dbCloudMonitor.InsertServerAlertDetail(model);
                }
                else
                {
                    //查询到了，更新次数
                    detailModel.预警说明 = model.预警说明;
                    detailModel.当前值 = model.当前值;
                    detailModel.预警次数 += model.预警次数;
                    detailModel.最后修改时间 = model.最后修改时间;
                    dbCloudMonitor.UpdateServerAlertDetailNumber(detailModel);
                }
            }
            return model.预警ID;
        }

        /// <summary>
        /// 发送通知
        /// </summary>
        /// <param name="alertlId">预警ID</param>
        private void SendNotice(object alertlId)
        {
            //获取数据，处理数据
            var alertModel = GetServerAlertByAlertlId((string)alertlId);
            var toUsers = new List<string>();
            var openIdList = new List<string>();
            var phoneList = new List<string>();
            var ccUsers = new List<string>();

            //消息分级，红色才抄送总部
            if (alertModel.预警等级 == (int)AlertLevelEnum.红色预警)
            {
                //处理总部配置
                var config = GetCloudMonitorConfig().AlertConfig;
                if (config.邮件推送开关 == "on")
                {
                    var mailStr = config.邮箱推送账号;
                    if (!string.IsNullOrEmpty(mailStr))
                    {
                        ccUsers.AddRange(mailStr.Split(';'));
                    }
                }
                if (config.微信消息开关 == "on")
                {
                    var phoneStr = config.微信推送帐号;
                    if (!string.IsNullOrEmpty(phoneStr))
                    {
                        phoneList.AddRange(phoneStr.Split(';'));
                    }
                }
            }

            //处理项目上的邮件推送和微信推送
            var itemInfo = dbCloudMonitor.GetProInfo(alertModel.项目ID);
            if (itemInfo != null && itemInfo.Rows.Count > 0)
            {
                var mailStr = itemInfo.Rows[0].GetValueByName<string>("邮件推送");
                if (!string.IsNullOrEmpty(mailStr))
                {
                    toUsers.AddRange(mailStr.Split(';'));
                }
                var phoneStr = itemInfo.Rows[0].GetValueByName<string>("微信推送");
                if (!string.IsNullOrEmpty(phoneStr))
                {
                    phoneList.AddRange(phoneStr.Split(';'));
                }
            }

            //处理渠道上的
            itemInfo = dbCloudMonitor.GetChannelInfo(alertModel.渠道ID);
            if (itemInfo != null && itemInfo.Rows.Count > 0)
            {
                var mailStr = itemInfo.Rows[0].GetValueByName<string>("技术联系人邮箱");
                if (!string.IsNullOrEmpty(mailStr))
                {
                    ccUsers.AddRange(mailStr.Split(';'));
                }
                var phoneStr = itemInfo.Rows[0].GetValueByName<string>("技术联系人微信");
                if (!string.IsNullOrEmpty(phoneStr))
                {
                    phoneList.AddRange(phoneStr.Split(';'));
                }

                //消息分级，红色才抄送渠道负责人
                if (alertModel.预警等级 == (int)AlertLevelEnum.红色预警)
                {
                    mailStr = itemInfo.Rows[0].GetValueByName<string>("商务联系人邮箱");
                    if (!string.IsNullOrEmpty(mailStr))
                    {
                        ccUsers.AddRange(mailStr.Split(';'));
                    }

                    phoneStr = itemInfo.Rows[0].GetValueByName<string>("商务联系人微信");
                    if (!string.IsNullOrEmpty(phoneStr))
                    {
                        phoneList.AddRange(phoneStr.Split(';'));
                    }
                }

            }

            //根据手机号获取openid
            openIdList = dbCloudMonitor.GetUserOpenIdByPhone(phoneList);

            var 处理状态 = alertModel.处理时间 == DateTime.MinValue ? "未处理" : "已处理";
            var title = $"【云监控】{alertModel.预警类型显示}预警{处理状态}";
            var url = $"{publishingAddress}/CloudMonitor/EarlyWarningInfo?id={alertModel.ID}";
            var body = $@"渠道名称：{alertModel.渠道名称}
项目名称：{alertModel.项目名称}
服务器IP：{alertModel.服务器IP}
负责人：{alertModel.负责人}
预警类型：{alertModel.预警类型显示}
预警等级：{alertModel.预警等级显示}
异常说明：{alertModel.异常说明}
处理状态：{处理状态}";
            if (处理状态 == "已处理")
            {
                body += $@"
处理说明：{alertModel.处理说明}";
            }

            SendMail(title, body, url, toUsers, ccUsers);
            Log.Info("开始准备发送微信消息：" + Strings.ObjectToJson(openIdList));
            SendWxMsg(title, url, openIdList);
        }

        /// <summary>
        /// 发送邮件
        /// </summary>
        /// <param name="title">邮件标题</param>
        /// <param name="body">邮件内容</param>
        /// <param name="toUsers">发送给谁</param>
        /// <param name="ccUsers">抄送给谁</param>
        private void SendMail(string title, string body, string url, List<string> toUsers, List<string> ccUsers)
        {
            if (toUsers == null) return;
            toUsers = toUsers.Distinct().ToList();
            toUsers = toUsers.Where(m => !string.IsNullOrEmpty(m)).ToList();
            if (toUsers.Count == 0) return;
            if (ccUsers == null) ccUsers = new List<string>();
            ccUsers = ccUsers.Distinct().ToList();
            ccUsers = ccUsers.Where(m => !string.IsNullOrEmpty(m)).ToList();


            var emailConfig = configBll.GetEmailConfig();
            var emailServerInfo = new QWPlatform.SystemLibrary.Email.EmailServerInfo(emailConfig.EmailServer, emailConfig.SendName, emailConfig.SendEmail, emailConfig.SendPassword, emailConfig.Port);

            //执行邮件发送
            QWPlatform.SystemLibrary.Email.SmtpEmail smtpEmail = new QWPlatform.SystemLibrary.Email.SmtpEmail(emailServerInfo);

            var emailInfo = new QWPlatform.SystemLibrary.Email.EmailInfo(title, body + $"\r\n\r\n查看详情：{url}", toUsers.ToArray(), ccUsers.ToArray());
            emailInfo.FromUser(emailConfig.SendEmail, emailConfig.SendName);

            var r = smtpEmail.Send(emailInfo);
            if (!r)
            {
                //送信失败
                Log.Warn("邮件发送失败，请检查密码或服务器配置是否正确");
            }
        }

        /// <summary>
        /// 发送微信消息
        /// </summary>
        /// <param name="title"></param>
        /// <param name="body"></param>
        /// <param name="openIdList"></param>
        private void SendWxMsg(string title, string url, List<string> openIdList)
        {
            if (openIdList == null) return;
            openIdList = openIdList.Distinct().ToList();
            openIdList = openIdList.Where(m => !string.IsNullOrEmpty(m)).ToList();
            if (openIdList.Count == 0) return;
            List<MessageDetailsDto> list = openIdList.Select(openid => new MessageDetailsDto() { 接收人id = openid }).ToList();
            NotificationDto no = new NotificationDto
            {
                style = "3",
                template_code = "OIE51OsXeECnoPxGgPTQ7acXybc7t1plgAbdlH-mMWE",
                template_value = "{\"first\":{\"value\":\"" + title + "\",\"color\":\"#173177\"},\"keyword1\":{\"value\":\"" + DateTime.Now + "\",\"color\":\"#173177\"},\"keyword2\":{\"value\":\"" + title + "\",\"color\":\"#173177\"},\"remark\":{\"value\":\"点击【查看详情】查看问题\",\"color\":\"#173177\"}}",
                url = url,
                record_items = list
            };
            Log.Info("发送微信消息：" + Strings.ObjectToJson(no));
            notice.SendWeChartTempletMessge(no);
        }

        /// <summary>
        /// 根据预警id获取服务器预警
        /// </summary>
        /// <returns></returns>
        public ServerAlertModel GetServerAlertByAlertlId(string alertlId)
        {
            var data = dbCloudMonitor.GetServerAlertByAlertlId(alertlId);
            WorkServerAlert(new List<ServerAlertModel>() { data });
            var 处理状态 = data.处理时间 == DateTime.MinValue ? "未处理" : "已处理";
            if (处理状态 == "已处理")
            {
                var processList = GetServerAlertProcessByAlertId(data.ID);
                data.处理说明 = processList.FirstOrDefault(m => m.处理状态 == (int)AlertStateEnum.已处理)?.处理说明;
            }
            return data;
        }

        /// <summary>
        /// 根据预警id获取服务器预警
        /// </summary>
        /// <returns></returns>
        public List<ServerAlertModel> GetServerAlertByAlertlId(List<string> alertlId)
        {
            var data = dbCloudMonitor.GetServerAlertByAlertlId(alertlId);
            WorkServerAlert(data);
            return data;
        }

        /// <summary>
        /// 获取所有服务器预警
        /// </summary>
        /// <returns></returns>
        public List<ServerAlertModel> GetServerAlertAll(string search)
        {
            var data = dbCloudMonitor.GetServerAlertAll(search);
            WorkServerAlert(data);
            return data;
        }

        /// <summary>
        /// 根据条件获取服务器预警
        /// </summary>
        /// <returns></returns>
        public List<ServerAlertModel> GetServerAlertBySearch(ServerAlertSearch search)
        {
            var config = GetCloudMonitorConfig().AlertConfig;
            var data = dbCloudMonitor.GetServerAlertBySearch(search, config);
            WorkServerAlert(data);
            return data;
        }

        /// <summary>
        /// 处理服务器预警信息
        /// </summary>
        /// <param name="data"></param>
        private void WorkServerAlert(List<ServerAlertModel> data)
        {
            // 定义一个批次大小
            int batchSize = 100;

            // 将数据拆分为多个批次
            IEnumerable<IEnumerable<string>> batches = data.Select((value, index) => new { value, index })
                .GroupBy(pair => pair.index / batchSize, pair => pair.value.ID)
                .Select(group => group.AsEnumerable());

            // 遍历每个批次并获取警报详情
            List<ServerAlertDetailModel> details = new List<ServerAlertDetailModel>();
            foreach (var batch in batches)
            {
                // 将批次中的 ID 列表转换为以逗号分隔的字符串
                string ids = string.Join(",", batch);

                // 确保 alertIds 不为空，然后获取相应的服务器警报详情
                if (!string.IsNullOrEmpty(ids))
                {
                    var detail = dbCloudMonitor.GetServerAlertDetailByAlertlId(ids);
                    details.AddRange(detail);
                }
            }

            //IEnumerable<string> list = data.Select(m => m.ID);
            //var alertIds = string.Join(",", list);
            //List<ServerAlertDetailModel> details = dbCloudMonitor.GetServerAlertDetailByAlertlId(alertIds);
            foreach (var item in data)
            {
                var curDetails = details.Where(m => item.ID == m.预警ID);
                //最新的一条记录
                var lasterDetail = curDetails.OrderByDescending(m => m.最后修改时间).FirstOrDefault();
                item.预警次数 = curDetails.Sum(m => m.预警次数);
                item.橙色预警次数 = curDetails.Where(m => m.预警等级 == (int)AlertLevelEnum.橙色预警).Sum(m => m.预警次数);
                item.红色预警次数 = curDetails.Where(m => m.预警等级 == (int)AlertLevelEnum.红色预警).Sum(m => m.预警次数);
                item.最后上报时间 = lasterDetail.最后修改时间;
                var lasterTitle = Enum.GetName(typeof(AlertLevelEnum), lasterDetail.预警等级);
                item.预警等级 = lasterDetail.预警等级;
                item.预警等级显示 = lasterTitle;
                switch (item.预警类型)
                {
                    case (int)AlertTypeEnum.硬件CPU:
                    case (int)AlertTypeEnum.硬件内存:
                    case (int)AlertTypeEnum.硬件磁盘空间:
                        item.异常说明 = $"{lasterTitle}，{lasterDetail.预警名称}预警值{lasterDetail.预警说明}，当前{lasterDetail.当前值}";
                        break;
                    case (int)AlertTypeEnum.数据库连接数不足:
                        DbConnectInfo dbConnectInfo = Strings.JsonToModel<DbConnectInfo>(lasterDetail.预警说明 ?? "");
                        item.异常说明 = $"{dbConnectInfo?.DbName}数据库连接数已达{dbConnectInfo.ConnectUsed}，最大{dbConnectInfo.ConnectTotal}";
                        break;
                    case (int)AlertTypeEnum.数据库死锁:
                        DbLockedInfo dbLockedInfo = Strings.JsonToModel<DbLockedInfo>(lasterDetail.预警说明 ?? "");
                        item.异常说明 = $"{dbLockedInfo?.DbName}数据库出现死锁，当前锁定数为：{lasterDetail.当前值}";
                        break;
                    case (int)AlertTypeEnum.服务中断:
                        var groups = curDetails.GroupBy(m => m.预警名称).Select(m => $"{m.Key}状态异常{m.Sum(n => n.预警次数)}次");
                        item.异常说明 = string.Join("\r\n", groups);
                        break;
                    case (int)AlertTypeEnum.自定义指标:
                        DbCustomAlertInfo dbCustomAlertInfo = Strings.JsonToModel<DbCustomAlertInfo>(lasterDetail.预警说明 ?? "");
                        item.异常说明 = $"数据库：{dbCustomAlertInfo?.DbName},{dbCustomAlertInfo?.CustomName}：当前值为：{lasterDetail.当前值}";
                        break;
                    case (int)AlertTypeEnum.磁盘可用天数:
                        item.异常说明 = $"{lasterTitle}，{lasterDetail.预警名称}，当前剩余可用天数：{lasterDetail.当前值}";
                        break;
                    case (int)AlertTypeEnum.采集中断:
                        item.异常说明 = lasterDetail.预警名称 + " " + lasterDetail.预警说明 + " " + lasterDetail.当前值;
                        break;
                    case (int)AlertTypeEnum.自定义服务:
                        item.异常说明 = lasterDetail.预警名称 + " " + lasterDetail.预警说明 + " " + lasterDetail.当前值;
                        break;
                    default:
                        item.异常说明 = lasterDetail.预警名称 + " " + lasterDetail.预警说明 + " " + lasterDetail.当前值;
                        break;
                }
            }
        }

        #endregion

        #region 服务器预警处理
        /// <summary>
        /// 插入服务器预警处理
        /// </summary>
        /// <param name="model"></param>
        /// <returns></returns>
        public bool InsertServerAlertProcess(ServerAlertProcessModel model)
        {
            var idStr = model.预警ID;
            var ids = idStr.Split(',').ToList();
            foreach (var id in ids)
            {
                var newModel = new ServerAlertProcessModel()
                {
                    ID = Guid.NewGuid().ToString(),
                    预警ID = id,
                    创建时间 = DateTime.Now,
                    处理人 = model.处理人,
                    处理说明 = model.处理说明,
                    处理状态 = model.处理状态,
                };
                var data = dbCloudMonitor.InsertServerAlertProcess(newModel);
                dbCloudMonitor.ProcessServerAlert(newModel.预警ID, newModel.处理状态, newModel.创建时间);
            }
            if (model.处理状态 == (int)AlertStateEnum.已处理)
            {
                ThreadPool.QueueUserWorkItem(new WaitCallback(SendNotice), ids[0]);
            }
            return true;
        }

        /// <summary>
        /// 查询服务器预警处理过程
        /// </summary>
        /// <param name="alertId"></param>
        /// <returns></returns>
        public List<ServerAlertProcessModel> GetServerAlertProcessByAlertId(string alertId)
        {
            var data = dbCloudMonitor.GetServerAlertProcessByAlertId(alertId);
            return data;
        }

        #endregion

        #region 项目监控处理

        /// <summary>
        /// 启动或者停止项目监控
        /// </summary>
        /// <param name="projectId"></param>
        /// <param name="state"></param>
        public bool StartOrStopProjectMonitor(string projectId, bool state, string clientId = "")
        {
            _cacheProjectMonitorList = null;
            var info = dbCloudMonitor.GetProjectMonitorByProjectId(projectId);
            if (info == null)
            {
                info = new ProjectMonitorModel();
                info.ID = Guid.NewGuid().ToString("D");
                info.项目ID = projectId;
                info.首次监控日期 = DateTime.Now;
                info.最后监控日期 = info.首次监控日期;
                info.监控状态 = state ? 1 : 2;
                info.客户端ID = clientId;
                return dbCloudMonitor.InsertProjectMonitor(info);
            }
            else
            {
                info.最后监控日期 = DateTime.Now;
                info.监控状态 = state ? 1 : 2;
                info.客户端ID = clientId;
                return dbCloudMonitor.UpdateProjectMonitor(info);
            }
        }

        /// <summary>
        /// 获取状态为正常的项目监控数据
        /// </summary>
        /// <param name="projectId"></param>
        /// <returns></returns>
        public List<ProjectMonitorModel> GetProjectMonitor()
        {
            if (_cacheProjectMonitorList == null)
                _cacheProjectMonitorList = dbCloudMonitor.GetProjectMonitor();
            return _cacheProjectMonitorList;
        }

        /// <summary>
        /// 获取全部项目监控数据
        /// </summary>
        /// <returns></returns>
        public List<ProjectMonitorModel> GetProjectMonitorAll()
        {
            var data = dbCloudMonitor.GetProjectMonitorAll();
            return data;
        }

        /// <summary>
        /// 根据查询获取全部项目监控
        /// </summary>
        /// <returns></returns>
        public List<ProjectMonitorModel> GetProjectMonitorBySearch(ServerAlertSearch search)
        {
            var data = dbCloudMonitor.GetProjectMonitorBySearch(search);
            return data;
        }

        /// <summary>
        /// 获取项目的监控状态，根据渠道id过滤
        /// </summary>
        /// <param name="channelId"></param>
        /// <returns></returns>
        public List<ProjectMonitorModel> GetProjectMonitorByChannelId(string channelId)
        {
            var data = dbCloudMonitor.GetProjectMonitorByChannelId(channelId);
            return data;
        }

        #endregion

        #region 数据库管理

        /// <summary>
        /// 插入数据库数据
        /// </summary>
        /// <param name="model"></param>
        /// <returns></returns>
        public bool InsertDbInfo(DbInfoModel model)
        {
            model.ID = Guid.NewGuid().ToString("D");
            var data = dbCloudMonitor.InsertDbInfo(model);
            return data;
        }

        /// <summary>
        /// 修改数据库数据
        /// </summary>
        /// <param name="model"></param>
        /// <returns></returns>
        public bool UpdateDbInfo(DbInfoModel model)
        {
            var data = dbCloudMonitor.UpdateDbInfo(model);
            return data;
        }

        /// <summary>
        /// 获取数据库数据，根据服务器ID过滤
        /// </summary>
        /// <param name="channelId"></param>
        /// <returns></returns>
        public List<DbInfoModel> GetDbInfoByServerId(string serverId)
        {
            var data = dbCloudMonitor.GetDbInfoByServerId(serverId);
            return data;
        }

        /// <summary>
        /// 获取数据库数据，根据ID过滤
        /// </summary>
        /// <param name="channelId"></param>
        /// <returns></returns>
        public DbInfoModel GetDbInfoById(string id)
        {
            var data = dbCloudMonitor.GetDbInfoById(id);
            return data;
        }

        /// <summary>
        /// 删除数据库数据，根据ID
        /// </summary>
        /// <param name="id"></param>
        /// <returns></returns>
        public bool DeleteDbInfoById(string id)
        {
            var data = dbCloudMonitor.DeleteDbInfoById(id);
            return data;
        }

        #endregion

        #region 上报指标记录

        /// <summary>
        /// 插入数据库数据
        /// </summary>
        /// <param name="model"></param>
        /// <returns></returns>
        public bool InsertPushIndexDataByServerId(PushIndexDataModel model)
        {
            var data = dbCloudMonitor.InsertPushIndexDataByServerId(model);
            return data;
        }

        /// <summary>
        /// 查询上传指标数据
        /// </summary>
        /// <returns></returns>
        public PushIndexDataModelTablePage GetPushIndexData(IndexSearch search)
        {
            var data = dbCloudMonitor.GetPushIndexData(search);
            var total = dbCloudMonitor.GetPushIndexDataTotal(search);
            PushIndexDataModelTablePage pushIndexDataModelTablePage = new PushIndexDataModelTablePage();
            pushIndexDataModelTablePage.total = total;
            pushIndexDataModelTablePage.rows = data;
            return pushIndexDataModelTablePage;
        }

        #endregion

        #region 服务器信息上报

        /// <summary>
        /// 接收上报的服务器信息
        /// </summary>
        /// <param name="serverInfos"></param>
        public void ReceiveServerInfo(string clientId, List<Ret<ServerInfo>> serverInfos)
        {
            //如果这个客户端ID没有启动，直接退出
            if (!GetProjectMonitor().Any(m => m.客户端ID == clientId)) return;
            if (!WorkTimeCheckContinue(clientId + "ReceiveServerInfo")) return;
            foreach (var item in serverInfos)
            {
                if (!item.Success)
                {
                    //上报指标记录
                    InsertPushIndexDataByServerId(new PushIndexDataModel()
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        服务器ID = item.Id,
                        预警ID = "",
                        类型 = (int)InfoTypeEnum.服务器硬件信息,
                        状态 = (int)PushStateTypeEnum.异常,
                        上报原始数据 = item.Msg,
                        创建时间 = item.CurrentTime,
                    });
                    continue;
                }
                var s = item.Data;
                ServerIndexInfoModel serverInfoModel = new ServerIndexInfoModel();
                serverInfoModel.最后修改时间 = s.CurrentTime;
                serverInfoModel.服务器ID = s.Id;
                serverInfoModel.CPU使用率 = s.CpuOccupancyRate;
                serverInfoModel.内存使用率 = s.MemoryUsed / s.MemoryTotal * 100;
                serverInfoModel.磁盘使用率 = s.DiskUsed / s.DiskTotal * 100;

                //处理预警
                var alertIds = ServerInfoAlert(serverInfoModel);

                //上报指标记录
                InsertPushIndexDataByServerId(new PushIndexDataModel()
                {
                    ID = Guid.NewGuid().ToString("D"),
                    服务器ID = item.Id,
                    预警ID = string.Join(",", alertIds),
                    类型 = (int)InfoTypeEnum.服务器硬件信息,
                    状态 = (int)PushStateTypeEnum.正常,
                    上报原始数据 = Strings.ObjectToJson(s),
                    创建时间 = item.CurrentTime,
                });

                //取缓存的指标id
                if (_cacheIndexId.TryGetValue(s.Id, out string indexId))
                {
                    //修改
                    serverInfoModel.ID = indexId;
                    dbCloudMonitor.UpdateServerInfo(serverInfoModel);
                }
                else
                {
                    //没取到，尝试从服务器取
                    var model = dbCloudMonitor.GetServerInfoByServerId(s.Id);
                    if (model == null)
                    {
                        //新增 新增要查询出对应的渠道id 项目id
                        indexId = Guid.NewGuid().ToString("D");
                        serverInfoModel.ID = indexId;
                        dbCloudMonitor.InsertServerInfo(serverInfoModel);
                    }
                    else
                    {
                        //修改
                        indexId = model.ID;
                        serverInfoModel.ID = indexId;
                        dbCloudMonitor.UpdateServerInfo(serverInfoModel);
                    }
                    _cacheIndexId.Add(s.Id, indexId);
                }

                //处理指标详情
                ServerIndexDetailModel detailModel = new ServerIndexDetailModel();
                detailModel.ID = Guid.NewGuid().ToString("D");
                detailModel.指标ID = indexId;
                detailModel.服务器ID = s.Id;
                detailModel.CPU使用率 = s.CpuOccupancyRate;
                detailModel.内存使用 = s.MemoryUsed;
                detailModel.内存可用 = s.MemoryTotal - s.MemoryUsed;
                detailModel.磁盘使用 = s.DiskUsed;
                detailModel.磁盘可用 = s.DiskTotal - s.DiskUsed;
                detailModel.创建时间 = s.CurrentTime;
                dbCloudMonitor.InsertServerDetail(detailModel);
            }
        }

        /// <summary>
        /// 自动处理预警（指标正常后）
        /// </summary>
        /// <param name="外部唯一键"></param>
        private void WithdrawAlert(AutomaticWarningProcessingModel automaticWarning)
        {
            var config = GetCloudMonitorConfig();
            //持续时间 单位次 大白话：多少次连续预警后xxx
            var duration = 1;
            //频率 定时任务的执行频率，两个时间相减 并在执行频率内，则为持续时间
            var frequency = 1;
            switch (automaticWarning.预警类型)
            {
                case (int)AlertTypeEnum.硬件CPU:
                    duration = config.AlertConfig.CPU预警持续时间;
                    frequency = config.CollectionConfig.服务器硬件信息执行频率;
                    break;
                case (int)AlertTypeEnum.硬件内存:
                    duration = config.AlertConfig.内存预警持续时间;
                    frequency = config.CollectionConfig.服务器硬件信息执行频率;
                    break;
                case (int)AlertTypeEnum.硬件磁盘空间:
                    duration = config.AlertConfig.磁盘预警续时间;
                    frequency = config.CollectionConfig.服务器硬件信息执行频率;
                    break;
                case (int)AlertTypeEnum.数据库连接数不足: break;
                case (int)AlertTypeEnum.数据库死锁: break;
                case (int)AlertTypeEnum.服务中断:
                    duration = config.AlertConfig.后台服务中断高危;
                    frequency = config.CollectionConfig.服务运行状态执行频率;
                    break;
                case (int)AlertTypeEnum.自定义指标: break;
                case (int)AlertTypeEnum.采集中断:
                    duration = config.AlertConfig.采集服务中断高危;
                    frequency = config.CollectionConfig.采集服务中断执行频率;
                    break;
                default:
                    break;
            }

            //持续时间等于1，没啥需要处理的了，直接过
            if (duration == 1)
            {
                WithdrawProcessingAlert(automaticWarning.ID);
                return;
            }


            //取上次预警的数据
            _cacheDuration.TryGetValue(automaticWarning.ID, out var model);
            //取到了判断是不是连续的，是连续的就把连续的+1
            if (model != null)
            {
                var newModel = (AutomaticWarningProcessingModel)model;
                var time = newModel.最后修改时间;
                var timeSpan = automaticWarning.最后修改时间 - newModel.最后修改时间;
                var minutes = timeSpan.TotalMinutes;
                //对比两次时间和频率是相等的，则认为是连续的
                //允许误差0.2分钟以内也认为是连续的
                if (frequency - 0.2 <= minutes || minutes <= frequency + 0.2)
                {
                    automaticWarning.连续预警次数 = newModel.连续预警次数 + 1;
                }
            }

            //预警次数大于指定的，就移除缓存
            //因为默认数字从0开始的，但是次数明显从1开始才合理，所以-1
            if (automaticWarning.连续预警次数 >= duration - 1)
            {
                _cacheDuration.Remove(automaticWarning.ID);
                WithdrawProcessingAlert(automaticWarning.ID);
                //预警
                return;
            }
            else
            {
                _cacheDuration[automaticWarning.ID] = automaticWarning;
            }
        }

        /// <summary>
        /// 处理撤回预警
        /// </summary>
        /// <param name="外部唯一键"></param>
        void WithdrawProcessingAlert(string 外部唯一键)
        {
            var serverAlert = dbCloudMonitor.GetServerAlertByServerIdAndAlertType(外部唯一键);
            if (serverAlert != null)
            {
                var newModel = new ServerAlertProcessModel()
                {
                    ID = Guid.NewGuid().ToString(),
                    预警ID = serverAlert.ID,
                    创建时间 = DateTime.Now,
                    处理人 = "自动撤回",
                    处理说明 = "自动撤回预警，状态已恢复",
                    处理状态 = 4,
                };
                var data = dbCloudMonitor.InsertServerAlertProcess(newModel);
                dbCloudMonitor.ProcessServerAlert(newModel.预警ID, newModel.处理状态, newModel.创建时间);
                ThreadPool.QueueUserWorkItem(new WaitCallback(SendNotice), serverAlert.ID);
            }
        }

        /// <summary>
        /// 服务信息预警处理
        /// </summary>
        /// <param name="serverInfo"></param>
        private List<string> ServerInfoAlert(ServerIndexInfoModel serverInfoModel)
        {
            var config = GetCloudMonitorConfig().AlertConfig;
            var currentValue = Math.Round(serverInfoModel.CPU使用率, 2);

            var alertIds = new List<string>();

            //CPU处理
            if (config.CPU使用濒危 <= currentValue && currentValue < config.CPU使用高危)
            {
                //橙色预警
                ServerAlertDetailModel model = new ServerAlertDetailModel
                {
                    ID = Guid.NewGuid().ToString("D"),
                    预警次数 = 1,
                    预警名称 = "CPU使用率",
                    预警说明 = config.CPU使用濒危 + "%",
                    当前值 = currentValue + "%",
                    预警等级 = (int)AlertLevelEnum.橙色预警,
                    最后修改时间 = serverInfoModel.最后修改时间,
                    服务器ID = serverInfoModel.服务器ID,
                    预警类型 = (int)AlertTypeEnum.硬件CPU,
                };
                model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
                //生成预警
                var alertId = CheckDuration(model, InsertServerAlert);
                if (!string.IsNullOrEmpty(alertId))
                {
                    alertIds.Add(alertId);
                }
            }
            else if (config.CPU使用高危 <= currentValue)
            {
                //红色预警
                ServerAlertDetailModel model = new ServerAlertDetailModel
                {
                    ID = Guid.NewGuid().ToString("D"),
                    预警次数 = 1,
                    预警名称 = "CPU使用率",
                    预警说明 = config.CPU使用高危 + "%",
                    当前值 = currentValue + "%",
                    预警等级 = (int)AlertLevelEnum.红色预警,
                    最后修改时间 = serverInfoModel.最后修改时间,
                    服务器ID = serverInfoModel.服务器ID,
                    预警类型 = (int)AlertTypeEnum.硬件CPU
                };
                model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
                //生成预警
                var alertId = CheckDuration(model, InsertServerAlert);
                if (!string.IsNullOrEmpty(alertId))
                {
                    alertIds.Add(alertId);
                }
            }
            else
            {
                var 外部唯一键 = (int)AlertTypeEnum.硬件CPU + "_" + serverInfoModel.服务器ID;
                AutomaticWarningProcessingModel auto = new AutomaticWarningProcessingModel
                {
                    ID = 外部唯一键,
                    预警类型 = (int)AlertTypeEnum.硬件CPU,
                    预警次数 = 1,
                    最后修改时间 = serverInfoModel.最后修改时间
                };
                WithdrawAlert(auto);
            }

            //内存处理
            currentValue = Math.Round(serverInfoModel.内存使用率, 2);
            if (config.内存使用濒危 <= currentValue && currentValue < config.内存使用高危)
            {
                //橙色预警
                ServerAlertDetailModel model = new ServerAlertDetailModel
                {
                    ID = Guid.NewGuid().ToString("D"),
                    预警次数 = 1,
                    预警名称 = "内存使用率",
                    预警说明 = config.内存使用濒危 + "%",
                    当前值 = currentValue + "%",
                    预警等级 = (int)AlertLevelEnum.橙色预警,
                    最后修改时间 = serverInfoModel.最后修改时间,
                    服务器ID = serverInfoModel.服务器ID,
                    预警类型 = (int)AlertTypeEnum.硬件内存
                };
                model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
                var alertId = CheckDuration(model, InsertServerAlert);
                if (!string.IsNullOrEmpty(alertId))
                {
                    alertIds.Add(alertId);
                }
            }
            else if (config.内存使用高危 <= currentValue)
            {
                //红色预警
                ServerAlertDetailModel model = new ServerAlertDetailModel
                {
                    ID = Guid.NewGuid().ToString("D"),
                    预警次数 = 1,
                    预警名称 = "内存使用率",
                    预警说明 = config.内存使用高危 + "%",
                    当前值 = currentValue + "%",
                    预警等级 = (int)AlertLevelEnum.红色预警,
                    最后修改时间 = serverInfoModel.最后修改时间,
                    服务器ID = serverInfoModel.服务器ID,
                    预警类型 = (int)AlertTypeEnum.硬件内存
                };
                model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
                var alertId = CheckDuration(model, InsertServerAlert);
                if (!string.IsNullOrEmpty(alertId))
                {
                    alertIds.Add(alertId);
                }
            }
            else
            {
                var 外部唯一键 = (int)AlertTypeEnum.硬件内存 + "_" + serverInfoModel.服务器ID;
                AutomaticWarningProcessingModel auto = new AutomaticWarningProcessingModel
                {
                    ID = 外部唯一键,
                    预警类型 = (int)AlertTypeEnum.硬件内存,
                    预警次数 = 1,
                    最后修改时间 = serverInfoModel.最后修改时间
                };
                WithdrawAlert(auto);
            }

            //磁盘处理
            currentValue = Math.Round(serverInfoModel.磁盘使用率, 2);
            if (config.磁盘使用濒危 <= currentValue && currentValue < config.磁盘使用高危)
            {
                //橙色预警
                ServerAlertDetailModel model = new ServerAlertDetailModel
                {
                    ID = Guid.NewGuid().ToString("D"),
                    预警次数 = 1,
                    预警名称 = "磁盘使用率",
                    预警说明 = config.磁盘使用濒危 + "%",
                    当前值 = currentValue + "%",
                    预警等级 = (int)AlertLevelEnum.橙色预警,
                    最后修改时间 = serverInfoModel.最后修改时间,
                    服务器ID = serverInfoModel.服务器ID,
                    预警类型 = (int)AlertTypeEnum.硬件磁盘空间
                };
                model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
                var alertId = CheckDuration(model, InsertServerAlert);
                if (!string.IsNullOrEmpty(alertId))
                {
                    alertIds.Add(alertId);
                }
            }
            else if (config.磁盘使用高危 <= currentValue)
            {
                //红色预警
                ServerAlertDetailModel model = new ServerAlertDetailModel
                {
                    ID = Guid.NewGuid().ToString("D"),
                    预警次数 = 1,
                    预警名称 = "磁盘使用率",
                    预警说明 = config.磁盘使用高危 + "%",
                    当前值 = currentValue + "%",
                    预警等级 = (int)AlertLevelEnum.红色预警,
                    最后修改时间 = serverInfoModel.最后修改时间,
                    服务器ID = serverInfoModel.服务器ID,
                    预警类型 = (int)AlertTypeEnum.硬件磁盘空间
                };
                model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
                var alertId = CheckDuration(model, InsertServerAlert);
                if (!string.IsNullOrEmpty(alertId))
                {
                    alertIds.Add(alertId);
                }
            }
            else
            {
                var 外部唯一键 = (int)AlertTypeEnum.硬件磁盘空间 + "_" + serverInfoModel.服务器ID;
                AutomaticWarningProcessingModel auto = new AutomaticWarningProcessingModel
                {
                    ID = 外部唯一键,
                    预警类型 = (int)AlertTypeEnum.硬件磁盘空间,
                    预警次数 = 1,
                    最后修改时间 = serverInfoModel.最后修改时间
                };
                WithdrawAlert(auto);
            }

            return alertIds;
        }

        /// <summary>
        /// 检查持续时间
        /// </summary>
        /// <param name="alertModel"></param>
        /// <param name="action"></param>
        /// <returns>预警ID,没值代表没预警</returns>
        private string CheckDuration(ServerAlertDetailModel alertModel, Func<ServerAlertDetailModel, string> action)
        {
            var cacheId = alertModel.预警名称 + alertModel.外部唯一键;
            var config = GetCloudMonitorConfig();
            //持续时间 单位次 大白话：多少次连续预警后xxx
            var duration = 1;
            //频率 定时任务的执行频率，两个时间相减 并在执行频率内，则为持续时间
            var frequency = 1;
            switch (alertModel.预警类型)
            {
                case (int)AlertTypeEnum.硬件CPU:
                    duration = config.AlertConfig.CPU预警持续时间;
                    frequency = config.CollectionConfig.服务器硬件信息执行频率;
                    break;
                case (int)AlertTypeEnum.硬件内存:
                    duration = config.AlertConfig.内存预警持续时间;
                    frequency = config.CollectionConfig.服务器硬件信息执行频率;
                    break;
                case (int)AlertTypeEnum.硬件磁盘空间:
                    duration = config.AlertConfig.磁盘预警续时间;
                    frequency = config.CollectionConfig.服务器硬件信息执行频率;
                    break;
                case (int)AlertTypeEnum.数据库连接数不足: break;
                case (int)AlertTypeEnum.数据库死锁: break;
                case (int)AlertTypeEnum.服务中断:
                    duration = config.AlertConfig.后台服务中断高危;
                    frequency = config.CollectionConfig.服务运行状态执行频率;
                    break;
                case (int)AlertTypeEnum.自定义指标: break;
                case (int)AlertTypeEnum.采集中断:
                    duration = config.AlertConfig.采集服务中断高危;
                    frequency = config.CollectionConfig.采集服务中断执行频率;
                    break;
                default:
                    break;
            }

            //持续时间等于1，没啥需要处理的了，直接过
            if (duration == 1)
            {
                //预警
                return action?.Invoke(alertModel);
            }


            //取上次预警的数据
            _cacheDuration.TryGetValue(cacheId, out var model);
            //取到了判断是不是连续的，是连续的就把连续的+1
            if (model != null)
            {
                var newModel = (ServerAlertDetailModel)model;
                var time = newModel.最后修改时间;
                var timeSpan = alertModel.最后修改时间 - newModel.最后修改时间;
                var minutes = timeSpan.TotalMinutes;
                //对比两次时间和频率是相等的，则认为是连续的
                //允许误差0.2分钟以内也认为是连续的
                if (frequency - 0.2 <= minutes || minutes <= frequency + 0.2)
                {
                    alertModel.连续预警次数 = newModel.连续预警次数 + 1;
                }
            }

            //预警次数大于指定的，就移除缓存
            //因为默认数字从0开始的，但是次数明显从1开始才合理，所以-1
            if (alertModel.连续预警次数 >= duration - 1)
            {
                _cacheDuration.Remove(cacheId);
                //预警
                return action?.Invoke(alertModel);
            }
            else
            {
                _cacheDuration[cacheId] = alertModel;
            }
            return "";
        }

        #endregion

        #region 数据库连接数信息上报

        /// <summary>
        /// 接收上报的数据库连接数信息
        /// </summary>
        /// <param name="dbConnectInfos"></param>
        /// <returns></returns>
        public void ReceiveDbConnectInfo(string clientId, List<Ret<DbConnectInfo>> dbConnectInfos)
        {
            //如果这个客户端ID没有启动，直接退出
            if (!GetProjectMonitor().Any(m => m.客户端ID == clientId)) return;
            if (!WorkTimeCheckContinue(clientId + "ReceiveDbConnectInfo")) return;
            var config = GetCloudMonitorConfig().AlertConfig;
            foreach (var item in dbConnectInfos)
            {
                if (!item.Success)
                {
                    //上报指标记录
                    InsertPushIndexDataByServerId(new PushIndexDataModel()
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        服务器ID = item.Id,
                        预警ID = "",
                        类型 = (int)InfoTypeEnum.数据库连接数,
                        状态 = (int)PushStateTypeEnum.异常,
                        上报原始数据 = item.Msg,
                        创建时间 = item.CurrentTime,
                        外部唯一键 = item.ExternalId
                    });
                    continue;
                }
                var info = item.Data;
                var alertId = "";

                var remain = info.ConnectTotal - info.ConnectUsed;
                if (config.数据库剩余连接高危 < remain && remain <= config.数据库剩余连接濒危)
                {
                    ServerAlertDetailModel model = new ServerAlertDetailModel
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        预警次数 = 1,
                        预警名称 = "数据库连接数不足",
                        预警说明 = Strings.ObjectToJson(info),
                        当前值 = remain.ToString(),
                        预警等级 = (int)AlertLevelEnum.橙色预警,
                        最后修改时间 = info.CurrentTime,
                        服务器ID = info.Id,
                        预警类型 = (int)AlertTypeEnum.数据库连接数不足
                    };
                    model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
                    alertId = CheckDuration(model, InsertServerAlert);
                }
                else if (remain <= config.数据库剩余连接高危)
                {
                    ServerAlertDetailModel model = new ServerAlertDetailModel
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        预警次数 = 1,
                        预警名称 = "数据库连接数不足",
                        预警说明 = Strings.ObjectToJson(info),
                        当前值 = remain.ToString(),
                        预警等级 = (int)AlertLevelEnum.红色预警,
                        最后修改时间 = info.CurrentTime,
                        服务器ID = info.Id,
                        预警类型 = (int)AlertTypeEnum.数据库连接数不足
                    };
                    model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
                    alertId = CheckDuration(model, InsertServerAlert);
                }

                //上报指标记录
                InsertPushIndexDataByServerId(new PushIndexDataModel()
                {
                    ID = Guid.NewGuid().ToString("D"),
                    服务器ID = item.Id,
                    预警ID = alertId,
                    类型 = (int)InfoTypeEnum.数据库连接数,
                    状态 = (int)PushStateTypeEnum.正常,
                    上报原始数据 = Strings.ObjectToJson(info),
                    创建时间 = item.CurrentTime,
                    外部唯一键 = item.ExternalId
                });
            }
        }

        #endregion

        #region 数据库死锁信息上报

        /// <summary>
        /// 接收上报的数据库死锁信息
        /// </summary>
        /// <param name="dbLockedInfos"></param>
        /// <returns></returns>
        public void ReceiveDbLockedInfo(string clientId, List<Ret<DbLockedInfo>> dbLockedInfos)
        {
            //如果这个客户端ID没有启动，直接退出
            if (!GetProjectMonitor().Any(m => m.客户端ID == clientId)) return;
            if (!WorkTimeCheckContinue(clientId + "ReceiveDbLockedInfo")) return;
            foreach (var item in dbLockedInfos)
            {
                if (!item.Success)
                {
                    //上报指标记录
                    InsertPushIndexDataByServerId(new PushIndexDataModel()
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        服务器ID = item.Id,
                        预警ID = "",
                        类型 = (int)InfoTypeEnum.数据库死锁,
                        状态 = (int)PushStateTypeEnum.异常,
                        上报原始数据 = item.Msg,
                        创建时间 = item.CurrentTime,
                        外部唯一键 = item.ExternalId
                    });
                    continue;
                }
                var info = item.Data;
                var alertId = "";
                if (info.Lockused > 0)
                {
                    ServerAlertDetailModel model = new ServerAlertDetailModel
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        预警次数 = 1,
                        预警名称 = "数据库死锁",
                        预警说明 = Strings.ObjectToJson(info),
                        当前值 = info.Lockused.ToString(),
                        预警等级 = (int)AlertLevelEnum.红色预警,
                        最后修改时间 = info.CurrentTime,
                        服务器ID = info.Id,
                        预警类型 = (int)AlertTypeEnum.数据库死锁
                    };
                    model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
                    alertId = CheckDuration(model, InsertServerAlert);
                }

                //上报指标记录
                InsertPushIndexDataByServerId(new PushIndexDataModel()
                {
                    ID = Guid.NewGuid().ToString("D"),
                    服务器ID = item.Id,
                    预警ID = alertId,
                    类型 = (int)InfoTypeEnum.数据库死锁,
                    状态 = (int)PushStateTypeEnum.正常,
                    上报原始数据 = Strings.ObjectToJson(info),
                    创建时间 = item.CurrentTime,
                    外部唯一键 = item.ExternalId
                });
            }
        }

        #endregion

        #region 服务发现信息上报

        /// <summary>
        /// 接收上报的服务发现信息
        /// </summary>
        /// <returns></returns>
        public void ReceiveServiceWorkStateInfo(string clientId, List<Ret<ServiceWorkStateInfo>> serviceWorkStateInfos)
        {
            //如果这个客户端ID没有启动，直接退出
            if (!GetProjectMonitor().Any(m => m.客户端ID == clientId)) return;
            if (!WorkTimeCheckContinue(clientId + "ReceiveServiceWorkStateInfo")) return;
            foreach (var item in serviceWorkStateInfos)
            {
                if (!item.Success)
                {
                    //上报指标记录
                    InsertPushIndexDataByServerId(new PushIndexDataModel()
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        服务器ID = item.Id,
                        预警ID = "",
                        类型 = (int)InfoTypeEnum.服务运行状态,
                        状态 = (int)PushStateTypeEnum.异常,
                        上报原始数据 = item.Msg,
                        创建时间 = item.CurrentTime,
                    });
                    continue;
                }
                var info = item.Data;
                var alertId = "";
                foreach (var state in info?.ServiceWorkStates)
                {
                    if (state.Status != "passing")
                    {
                        ServerAlertDetailModel model = new ServerAlertDetailModel
                        {
                            ID = Guid.NewGuid().ToString("D"),
                            预警次数 = 1,
                            预警名称 = state.ServiceName,
                            预警说明 = Strings.ObjectToJson(state),
                            当前值 = state.Status,
                            预警等级 = (int)AlertLevelEnum.红色预警,
                            最后修改时间 = info.CurrentTime,
                            服务器ID = info.Id,
                            预警类型 = (int)AlertTypeEnum.服务中断
                        };
                        model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
                        alertId = CheckDuration(model, InsertServerAlert);
                    }
                }

                //上报指标记录
                InsertPushIndexDataByServerId(new PushIndexDataModel()
                {
                    ID = Guid.NewGuid().ToString("D"),
                    服务器ID = item.Id,
                    预警ID = alertId,
                    类型 = (int)InfoTypeEnum.服务运行状态,
                    状态 = (int)PushStateTypeEnum.正常,
                    上报原始数据 = Strings.ObjectToJson(info),
                    创建时间 = item.CurrentTime,
                });
            }
        }
        #endregion

        #region 自定义预警信息上报

        /// <summary>
        /// 接收上报的自定义预警信息
        /// </summary>
        /// <returns></returns>
        public void ReceiveDbCustomAlertInfo(string clientId, Ret<DbCustomAlertInfo> item)
        {
            //如果这个客户端ID没有启动，直接退出
            if (!GetProjectMonitor().Any(m => m.客户端ID == clientId)) return;
            if (!item.Success)
            {
                //上报指标记录
                InsertPushIndexDataByServerId(new PushIndexDataModel()
                {
                    ID = Guid.NewGuid().ToString("D"),
                    服务器ID = item.Id,
                    预警ID = "",
                    类型 = (int)InfoTypeEnum.自定义预警,
                    状态 = (int)PushStateTypeEnum.异常,
                    上报原始数据 = item.Msg,
                    创建时间 = item.CurrentTime,
                    外部唯一键 = item.ExternalId
                });
                return;
            }
            var info = item.Data;
            var alertId = "";

            //var customItem = GetCustomAlertModelByStart().FirstOrDefault(m => m.ID == info.CustomId);
            //if (customItem == null) return;

            if (info.Value != null)
            {
                ServerAlertDetailModel model = new ServerAlertDetailModel
                {
                    ID = Guid.NewGuid().ToString("D"),
                    预警次数 = 1,
                    预警名称 = info.CustomId,
                    预警说明 = Strings.ObjectToJson(info),
                    当前值 = info.Value,
                    预警等级 = (int)AlertLevelEnum.红色预警,
                    最后修改时间 = info.CurrentTime,
                    服务器ID = info.Id,
                    预警类型 = (int)AlertTypeEnum.自定义指标,
                };
                model.外部唯一键 = model.预警类型 + "_" + info.CustomId;
                alertId = CheckDuration(model, InsertServerAlert);
            }

            //上报指标记录
            InsertPushIndexDataByServerId(new PushIndexDataModel()
            {
                ID = Guid.NewGuid().ToString("D"),
                服务器ID = item.Id,
                预警ID = alertId,
                类型 = (int)InfoTypeEnum.自定义预警,
                状态 = (int)PushStateTypeEnum.正常,
                上报原始数据 = Strings.ObjectToJson(info),
                创建时间 = item.CurrentTime,
                外部唯一键 = item.ExternalId
            });
        }
        #endregion

        #region 自定义服务信息上报

        /// <summary>
        /// 接收上报的自定义服务信息
        /// </summary>
        /// <returns></returns>
        public void ReceiveCustomServiceInfo(string clientId, Ret<CustomServiceInfo> item)
        {
            //如果这个客户端ID没有启动，直接退出
            if (!GetProjectMonitor().Any(m => m.客户端ID == clientId)) return;
            if (!item.Success)
            {
                //上报指标记录
                InsertPushIndexDataByServerId(new PushIndexDataModel()
                {
                    ID = Guid.NewGuid().ToString("D"),
                    服务器ID = item.Id,
                    预警ID = "",
                    类型 = (int)InfoTypeEnum.自定义服务,
                    状态 = (int)PushStateTypeEnum.异常,
                    上报原始数据 = item.Msg,
                    创建时间 = item.CurrentTime,
                    外部唯一键 = item.ExternalId
                });
                return;
            }
            var info = item.Data;
            var alertId = "";

            if (info.Value != null)
            {
                try
                {
                    var customServiceInfo = dbCloudMonitor.GetCustomServiceInfoById(info.CustomServiceId);
                    var json = Strings.JsonToModel<CustomServiceAnalysis>(info.Value);
                    if (json.预警类型 <= 0) return;
                    ServerAlertDetailModel model = new ServerAlertDetailModel
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        预警次数 = 1,
                        预警名称 = customServiceInfo.服务名称,
                        预警说明 = Strings.ObjectToJson(info),
                        当前值 = info.Value,
                        预警等级 = json.预警类型,
                        项目ID = customServiceInfo.项目ID,
                        最后修改时间 = info.CurrentTime,
                        预警类型 = (int)AlertTypeEnum.自定义服务,
                    };
                    model.外部唯一键 = model.预警类型 + "_" + info.CustomServiceId;
                    alertId = CheckDuration(model, InsertServerAlert);
                }
                catch (Exception ex)
                {
                    //上报指标记录
                    InsertPushIndexDataByServerId(new PushIndexDataModel()
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        服务器ID = item.Id,
                        预警ID = "",
                        类型 = (int)InfoTypeEnum.自定义服务,
                        状态 = (int)PushStateTypeEnum.异常,
                        上报原始数据 = "处理数据异常:" + ex.Message + " 原始内容：" + item.Msg,
                        创建时间 = item.CurrentTime,
                        外部唯一键 = item.ExternalId
                    });
                    return;
                }
            }

            //上报指标记录
            InsertPushIndexDataByServerId(new PushIndexDataModel()
            {
                ID = Guid.NewGuid().ToString("D"),
                服务器ID = item.Id,
                预警ID = alertId,
                类型 = (int)InfoTypeEnum.自定义服务,
                状态 = (int)PushStateTypeEnum.正常,
                上报原始数据 = Strings.ObjectToJson(info),
                创建时间 = item.CurrentTime,
                外部唯一键 = item.ExternalId
            });
        }
        #endregion

        #region 服务器定时任务

        /// <summary>
        /// 数据清理
        /// </summary>
        public void DataCleaning()
        {
            try
            {
                Log.Info("数据清理：开始");
                var ret = dbCloudMonitor.DeleteServerIndexData();
                Log.Info($"数据清理：服务器指标明{ret}条");
                ret = dbCloudMonitor.DeletePushIndexData();
                Log.Info($"数据清理：上报指标记录{ret}条");
                Log.Info("数据清理：结束");
            }
            catch (Exception ex)
            {
                Log.Info("数据清理：错误:" + ex.Message);
            }
        }

        /// <summary>
        /// 磁盘可用天数扫描
        /// </summary>
        public void DiskAvailabilityScanning()
        {
            try
            {
                Log.Info("磁盘可用天数：开始");
                var nowTime = DateTime.Now;
                var lastTime = nowTime.AddDays(-1);
                var config = GetCloudMonitorConfig().AlertConfig;
                var projectList = GetProjectMonitor();
                if (!projectList.Any()) return;
                var projectIds = string.Join(",", projectList.Select(m => m.项目ID).ToList());
                var serverInfoList = GetServerByItemId(projectIds);
                foreach (var project in projectList)
                {
                    //服务器列表
                    var serverInfos = serverInfoList.Where(m => m.项目ID == project.项目ID);

                    var lastIndexData = dbCloudMonitor.GetPushIndexDataByServerInfo(project.项目ID, lastTime);
                    var nowIndexData = dbCloudMonitor.GetPushIndexDataByServerInfo(project.项目ID, nowTime);

                    foreach (var serverInfo in serverInfos)
                    {
                        var lastInedx = lastIndexData.FirstOrDefault(m => m.服务器ID == serverInfo.ID);
                        var nowInedx = nowIndexData.FirstOrDefault(m => m.服务器ID == serverInfo.ID);
                        if (lastInedx != null && nowInedx != null)
                        {
                            if (lastInedx.状态 != (int)PushStateTypeEnum.正常 || nowInedx.状态 != (int)PushStateTypeEnum.正常)
                            {
                                //状态有问题。可以选择预警一下
                                continue;
                            }
                            var lastInfo = Strings.JsonToModel<ServerInfo>(lastInedx.上报原始数据);
                            var nowInfo = Strings.JsonToModel<ServerInfo>(nowInedx.上报原始数据);
                            var oldDisk = lastInfo.DiskTotal - lastInfo.DiskUsed;
                            var nowDisk = nowInfo.DiskTotal - nowInfo.DiskUsed;

                            //一天使用的磁盘容量
                            var oneDayUse = oldDisk - nowDisk;
                            //小于10M的话，和没用没啥区别，不记录
                            if (oneDayUse > 10)
                            {
                                //可用天数
                                var availableDay = nowDisk / oneDayUse;
                                if (config.磁盘预计可用天数高危 < availableDay && availableDay <= config.磁盘预计可用天数濒危)
                                {
                                    ServerAlertDetailModel model = new ServerAlertDetailModel
                                    {
                                        ID = Guid.NewGuid().ToString("D"),
                                        预警次数 = 1,
                                        预警名称 = "磁盘可用天数不足",
                                        预警说明 = "",
                                        当前值 = availableDay.ToString(),
                                        预警等级 = (int)AlertLevelEnum.红色预警,
                                        最后修改时间 = DateTime.Now,
                                        服务器ID = serverInfo.ID,
                                        预警类型 = (int)AlertTypeEnum.磁盘可用天数
                                    };
                                    model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
                                    CheckDuration(model, InsertServerAlert);
                                }
                                else if (availableDay <= config.磁盘预计可用天数高危)
                                {
                                    ServerAlertDetailModel model = new ServerAlertDetailModel
                                    {
                                        ID = Guid.NewGuid().ToString("D"),
                                        预警次数 = 1,
                                        预警名称 = "磁盘可用天数不足",
                                        预警说明 = "",
                                        当前值 = availableDay.ToString(),
                                        预警等级 = (int)AlertLevelEnum.红色预警,
                                        最后修改时间 = DateTime.Now,
                                        服务器ID = serverInfo.ID,
                                        预警类型 = (int)AlertTypeEnum.磁盘可用天数
                                    };
                                    model.外部唯一键 = model.预警类型 + "_" + model.服务器ID;
                                    CheckDuration(model, InsertServerAlert);
                                }
                            }
                        }
                    }
                }
                Log.Info("磁盘可用天数：结束");
            }
            catch (Exception ex)
            {
                Log.Info("磁盘可用天数：错误:" + ex.Message);
            }

            //数据清理
            DataCleaning();
        }

        /// <summary>
        /// 采集中断扫描
        /// </summary>
        public void CollectionInterruptScanning()
        {
            try
            {
                Log.Info("采集中断：开始");
                var config = GetCloudMonitorConfig().CollectionConfig;
                var projectList = GetProjectMonitor();
                if (!projectList.Any()) return;
                var projectIds = string.Join(",", projectList.Select(m => m.项目ID).ToList());
                var serverInfoList = GetServerByItemId(projectIds);
                foreach (var project in projectList)
                {
                    var list = new List<ServerAlertDetailModel>();
                    var indexData = dbCloudMonitor.GetPushIndexDataByProjectId(project.项目ID, config.采集服务中断执行频率);
                    //服务器列表
                    var serverInfos = serverInfoList.Where(m => m.项目ID == project.项目ID);
                    list.AddRange(HandleServerInfo(serverInfos.ToList(), indexData.Where(m => m.类型 == (int)InfoTypeEnum.服务器硬件信息)));

                    //判断服务发现服务器
                    var serviceDiscoverys = serverInfos.Where(m => m.安装服务?.Split(',')?.Contains("7") == true).ToList();
                    list.AddRange(HandleServiceDiscovery(serviceDiscoverys, indexData.Where(m => m.类型 == (int)InfoTypeEnum.服务运行状态)));

                    //判断数据库服务器
                    var dbServers = serverInfos.Where(m => m.安装服务?.Split(',')?.Contains("9") == true).ToList();
                    list.AddRange(HandleDbConnect(dbServers, indexData.Where(m => m.类型 == (int)InfoTypeEnum.数据库连接数)));
                    list.AddRange(HandleDbLocked(dbServers, indexData.Where(m => m.类型 == (int)InfoTypeEnum.数据库死锁)));

                    var total = serverInfos.Count() + serviceDiscoverys.Count + (dbServers.Sum(m => m.数据库.Count) * 2);
                    //代表是整体中断
                    if (total == list.Count(m => m.预警说明 != "状态异常"))
                    {
                        CheckDuration(new ServerAlertDetailModel()
                        {
                            ID = Guid.NewGuid().ToString("D"),
                            预警次数 = 1,
                            预警名称 = "前置机中断",
                            预警说明 = "未能查询到上报数据",
                            当前值 = "",
                            预警等级 = (int)AlertLevelEnum.红色预警,
                            最后修改时间 = DateTime.Now,
                            服务器ID = "",
                            预警类型 = (int)AlertTypeEnum.采集中断,
                            项目ID = project.项目ID,
                            外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + project.项目ID
                        }, InsertServerAlert);
                    }
                    else
                    {
                        foreach (var item in list)
                        {
                            CheckDuration(item, InsertServerAlert);
                        }
                    }
                }
                Log.Info("采集中断：结束");
            }
            catch (Exception ex)
            {
                Log.Info("采集中断：错误:" + ex.Message);
            }
        }

        /// <summary>
        /// 处理服务器
        /// </summary>
        /// <param name="data"></param>
        /// <param name="indexData"></param>
        private List<ServerAlertDetailModel> HandleServerInfo(List<ServerInfoModel> data, IEnumerable<PushIndexDataModel> indexData)
        {
            var config = GetCloudMonitorConfig().CollectionConfig;
            var frequency = config.服务器硬件信息执行频率;
            var list = new List<ServerAlertDetailModel>();
            foreach (var serverInfo in data)
            {
                //获取当前服务器的并且最新的一条
                var lastIndex = indexData.Where(m => m.服务器ID == serverInfo.ID)
                                    .OrderByDescending(m => m.创建时间).FirstOrDefault();

                //一条指标都没查到，为服务中断
                if (lastIndex == null)
                {
                    list.Add(new ServerAlertDetailModel()
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        预警次数 = 1,
                        预警名称 = "服务器硬件信息中断",
                        预警说明 = "未能查询到上报数据",
                        当前值 = "",
                        预警等级 = (int)AlertLevelEnum.红色预警,
                        最后修改时间 = DateTime.Now,
                        服务器ID = serverInfo.ID,
                        预警类型 = (int)AlertTypeEnum.采集中断,
                        外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + (int)InfoTypeEnum.服务器硬件信息 + "_" + serverInfo.ID
                    });
                    continue;
                }

                var timeSpan = DateTime.Now - lastIndex.创建时间;
                var minutes = timeSpan.TotalMinutes;

                //对比两次时间和频率是相等的，则认为是连续的
                //允许误差0.2分钟以内也认为是连续的
                if (frequency - 0.2 <= minutes || minutes <= frequency + 0.2)
                {
                    //这儿是代表上报成功，继续处理数据是否有效
                    if (lastIndex.状态 == (int)PushStateTypeEnum.异常)
                    {
                        list.Add(new ServerAlertDetailModel()
                        {
                            ID = Guid.NewGuid().ToString("D"),
                            预警次数 = 1,
                            预警名称 = "服务器硬件信息中断",
                            预警说明 = "状态异常",
                            当前值 = $"异常说明:{lastIndex.上报原始数据}",
                            预警等级 = (int)AlertLevelEnum.红色预警,
                            最后修改时间 = DateTime.Now,
                            服务器ID = serverInfo.ID,
                            预警类型 = (int)AlertTypeEnum.采集中断,
                            外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + (int)InfoTypeEnum.服务器硬件信息 + "_" + serverInfo.ID
                        });
                    }
                }
                else
                {
                    //这儿是代表在预定的时间未能成功上报指标记录
                    list.Add(new ServerAlertDetailModel()
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        预警次数 = 1,
                        预警名称 = "服务器硬件信息中断",
                        预警说明 = "预定时间未能成功上报数据",
                        当前值 = $"上次上报时间：{lastIndex.创建时间:yyyy-MM-dd HH:mm:ss}",
                        预警等级 = (int)AlertLevelEnum.红色预警,
                        最后修改时间 = DateTime.Now,
                        服务器ID = serverInfo.ID,
                        预警类型 = (int)AlertTypeEnum.采集中断,
                        外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + (int)InfoTypeEnum.服务器硬件信息 + "_" + serverInfo.ID
                    });
                }
            }
            return list;
        }

        /// <summary>
        /// 处理服务发现
        /// </summary>
        private List<ServerAlertDetailModel> HandleServiceDiscovery(List<ServerInfoModel> data, IEnumerable<PushIndexDataModel> indexData)
        {
            var config = GetCloudMonitorConfig().CollectionConfig;
            var frequency = config.服务运行状态执行频率;
            var list = new List<ServerAlertDetailModel>();
            foreach (var serverInfo in data)
            {
                //获取当前服务器的并且最新的一条
                var lastIndex = indexData.Where(m => m.服务器ID == serverInfo.ID)
                                    .OrderByDescending(m => m.创建时间).FirstOrDefault();

                //一条指标都没查到，为服务中断
                if (lastIndex == null)
                {
                    list.Add(new ServerAlertDetailModel()
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        预警次数 = 1,
                        预警名称 = "服务运行状态中断",
                        预警说明 = "未能查询到上报数据",
                        当前值 = "",
                        预警等级 = (int)AlertLevelEnum.红色预警,
                        最后修改时间 = DateTime.Now,
                        服务器ID = serverInfo.ID,
                        预警类型 = (int)AlertTypeEnum.采集中断,
                        外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + (int)InfoTypeEnum.服务运行状态 + "_" + serverInfo.ID
                    });
                    continue;
                }

                var timeSpan = DateTime.Now - lastIndex.创建时间;
                var minutes = timeSpan.TotalMinutes;

                //对比两次时间和频率是相等的，则认为是连续的
                //允许误差0.2分钟以内也认为是连续的
                if (frequency - 0.2 <= minutes || minutes <= frequency + 0.2)
                {
                    //这儿是代表上报成功，继续处理数据是否有效
                    if (lastIndex.状态 == (int)PushStateTypeEnum.异常)
                    {
                        list.Add(new ServerAlertDetailModel()
                        {
                            ID = Guid.NewGuid().ToString("D"),
                            预警次数 = 1,
                            预警名称 = "服务运行状态中断",
                            预警说明 = "状态异常",
                            当前值 = $"异常说明:{lastIndex.上报原始数据}",
                            预警等级 = (int)AlertLevelEnum.红色预警,
                            最后修改时间 = DateTime.Now,
                            服务器ID = serverInfo.ID,
                            预警类型 = (int)AlertTypeEnum.采集中断,
                            外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + (int)InfoTypeEnum.服务运行状态 + "_" + serverInfo.ID
                        });
                    }
                }
                else
                {
                    //这儿是代表在预定的时间未能成功上报指标记录
                    list.Add(new ServerAlertDetailModel()
                    {
                        ID = Guid.NewGuid().ToString("D"),
                        预警次数 = 1,
                        预警名称 = "服务运行状态中断",
                        预警说明 = "预定时间未能成功上报数据",
                        当前值 = $"上次上报时间：{lastIndex.创建时间:yyyy-MM-dd HH:mm:ss}",
                        预警等级 = (int)AlertLevelEnum.红色预警,
                        最后修改时间 = DateTime.Now,
                        服务器ID = serverInfo.ID,
                        预警类型 = (int)AlertTypeEnum.采集中断,
                        外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + (int)InfoTypeEnum.服务运行状态 + "_" + serverInfo.ID
                    });
                }

            }
            return list;
        }

        /// <summary>
        /// 处理数据库连接
        /// </summary>
        /// <param name="data"></param>
        /// <param name="indexData"></param>
        private List<ServerAlertDetailModel> HandleDbConnect(List<ServerInfoModel> data, IEnumerable<PushIndexDataModel> indexData)
        {
            var config = GetCloudMonitorConfig().CollectionConfig;
            var frequency = config.数据库连接数执行频率;
            var list = new List<ServerAlertDetailModel>();
            foreach (var serverInfo in data)
            {

                foreach (var dbInfo in serverInfo.数据库)
                {
                    //获取当前服务器的并且最新的一条
                    var lastIndex = indexData.Where(m => m.服务器ID == serverInfo.ID)
                                        .Where(m => m.外部唯一键 == dbInfo.ID)
                                        .OrderByDescending(m => m.创建时间).FirstOrDefault();

                    //一条指标都没查到，为服务中断
                    if (lastIndex == null)
                    {
                        list.Add(new ServerAlertDetailModel()
                        {
                            ID = Guid.NewGuid().ToString("D"),
                            预警次数 = 1,
                            预警名称 = "数据库连接数中断",
                            预警说明 = "未能查询到上报数据",
                            当前值 = "",
                            预警等级 = (int)AlertLevelEnum.红色预警,
                            最后修改时间 = DateTime.Now,
                            服务器ID = serverInfo.ID,
                            预警类型 = (int)AlertTypeEnum.采集中断,
                            外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + (int)InfoTypeEnum.数据库连接数 + dbInfo.ID + "_" + serverInfo.ID
                        });
                        continue;
                    }

                    var timeSpan = DateTime.Now - lastIndex.创建时间;
                    var minutes = timeSpan.TotalMinutes;

                    //对比两次时间和频率是相等的，则认为是连续的
                    //允许误差0.2分钟以内也认为是连续的
                    if (frequency - 0.2 <= minutes || minutes <= frequency + 0.2)
                    {
                        //这儿是代表上报成功，继续处理数据是否有效
                        if (lastIndex.状态 == (int)PushStateTypeEnum.异常)
                        {
                            list.Add(new ServerAlertDetailModel()
                            {
                                ID = Guid.NewGuid().ToString("D"),
                                预警次数 = 1,
                                预警名称 = "数据库连接数中断",
                                预警说明 = "状态异常",
                                当前值 = $"异常说明:{lastIndex.上报原始数据}",
                                预警等级 = (int)AlertLevelEnum.红色预警,
                                最后修改时间 = DateTime.Now,
                                服务器ID = serverInfo.ID,
                                预警类型 = (int)AlertTypeEnum.采集中断,
                                外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + (int)InfoTypeEnum.数据库连接数 + dbInfo.ID + "_" + serverInfo.ID
                            });
                        }
                    }
                    else
                    {
                        //这儿是代表在预定的时间未能成功上报指标记录
                        list.Add(new ServerAlertDetailModel()
                        {
                            ID = Guid.NewGuid().ToString("D"),
                            预警次数 = 1,
                            预警名称 = "数据库连接数中断",
                            预警说明 = "预定时间未能成功上报数据",
                            当前值 = $"上次上报时间：{lastIndex.创建时间:yyyy-MM-dd HH:mm:ss}",
                            预警等级 = (int)AlertLevelEnum.红色预警,
                            最后修改时间 = DateTime.Now,
                            服务器ID = serverInfo.ID,
                            预警类型 = (int)AlertTypeEnum.采集中断,
                            外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + (int)InfoTypeEnum.数据库连接数 + dbInfo.ID + "_" + serverInfo.ID
                        });
                    }
                }
            }
            return list;
        }

        /// <summary>
        /// 处理数据库锁定
        /// </summary>
        /// <param name="data"></param>
        /// <param name="indexData"></param>
        private List<ServerAlertDetailModel> HandleDbLocked(List<ServerInfoModel> data, IEnumerable<PushIndexDataModel> indexData)
        {
            var config = GetCloudMonitorConfig().CollectionConfig;
            var frequency = config.数据库死锁执行频率;
            var list = new List<ServerAlertDetailModel>();
            foreach (var serverInfo in data)
            {

                foreach (var dbInfo in serverInfo.数据库)
                {
                    //获取当前服务器的并且最新的一条
                    var lastIndex = indexData.Where(m => m.服务器ID == serverInfo.ID)
                                        .Where(m => m.外部唯一键 == dbInfo.ID)
                                        .OrderByDescending(m => m.创建时间).FirstOrDefault();

                    //一条指标都没查到，为服务中断
                    if (lastIndex == null)
                    {
                        list.Add(new ServerAlertDetailModel()
                        {
                            ID = Guid.NewGuid().ToString("D"),
                            预警次数 = 1,
                            预警名称 = "数据库死锁中断",
                            预警说明 = "未能查询到上报数据",
                            当前值 = "",
                            预警等级 = (int)AlertLevelEnum.红色预警,
                            最后修改时间 = DateTime.Now,
                            服务器ID = serverInfo.ID,
                            预警类型 = (int)AlertTypeEnum.采集中断,
                            外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + (int)InfoTypeEnum.数据库死锁 + dbInfo.ID + "_" + serverInfo.ID
                        });
                        continue;
                    }

                    var timeSpan = DateTime.Now - lastIndex.创建时间;
                    var minutes = timeSpan.TotalMinutes;

                    //对比两次时间和频率是相等的，则认为是连续的
                    //允许误差0.2分钟以内也认为是连续的
                    if (frequency - 0.2 <= minutes || minutes <= frequency + 0.2)
                    {
                        //这儿是代表上报成功，继续处理数据是否有效
                        if (lastIndex.状态 == (int)PushStateTypeEnum.异常)
                        {
                            list.Add(new ServerAlertDetailModel()
                            {
                                ID = Guid.NewGuid().ToString("D"),
                                预警次数 = 1,
                                预警名称 = "数据库死锁中断",
                                预警说明 = "状态异常",
                                当前值 = $"异常说明:{lastIndex.上报原始数据}",
                                预警等级 = (int)AlertLevelEnum.红色预警,
                                最后修改时间 = DateTime.Now,
                                服务器ID = serverInfo.ID,
                                预警类型 = (int)AlertTypeEnum.采集中断,
                                外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + (int)InfoTypeEnum.数据库死锁 + dbInfo.ID + "_" + serverInfo.ID
                            });
                        }
                    }
                    else
                    {
                        //这儿是代表在预定的时间未能成功上报指标记录
                        list.Add(new ServerAlertDetailModel()
                        {
                            ID = Guid.NewGuid().ToString("D"),
                            预警次数 = 1,
                            预警名称 = "数据库死锁中断",
                            预警说明 = "预定时间未能成功上报数据",
                            当前值 = $"上次上报时间：{lastIndex.创建时间:yyyy-MM-dd HH:mm:ss}",
                            预警等级 = (int)AlertLevelEnum.红色预警,
                            最后修改时间 = DateTime.Now,
                            服务器ID = serverInfo.ID,
                            预警类型 = (int)AlertTypeEnum.采集中断,
                            外部唯一键 = (int)AlertTypeEnum.采集中断 + "_" + (int)InfoTypeEnum.数据库死锁 + dbInfo.ID + "_" + serverInfo.ID
                        });
                    }
                }
            }

            return list;
        }

        #endregion

        #region 工作时间
        /// <summary>
        /// 表示非工作时间的采集频率
        /// </summary>
        private const int workFrequency = 15;
        private static Dictionary<string, int> _cacheWorkTime = new Dictionary<string, int>();

        /// <summary>
        /// 判断工作时间，是否继续
        /// </summary>
        /// <param name="id"></param>
        /// <returns></returns>
        private bool WorkTimeCheckContinue(string id)
        {
            //不是工作时间直接继续
            if (!IsWorkTime()) return true;
            if (_cacheWorkTime.TryGetValue(id, out var num))
            {
                num++;
                if (num >= workFrequency)
                {
                    _cacheWorkTime[id] = 0;
                    return true;
                }
            }
            else
            {
                _cacheWorkTime[id] = 1;
            }

            return false;
        }

        /// <summary>
        /// 判断当前时间是否工作时间
        /// </summary>
        /// <returns></returns>
        private bool IsWorkTime()
        {
            var config = GetCloudMonitorConfig();
            AlertConfig alertConfig = config.AlertConfig;

            // 当前时间的小时和分钟
            var currentTime = DateTime.Now.TimeOfDay;

            // 将字符串时间转换为 TimeSpan
            var startTime = TimeSpan.Parse(alertConfig.工作开始时间);
            var endTime = TimeSpan.Parse(alertConfig.工作结束时间);

            // 检查当前时间是否在时间区间内
            if (currentTime >= startTime && currentTime <= endTime)
            {
                return true;
            }
            else
            {
                return false;
            }
        }
        #endregion
    }
}
;