接口有效期控制与API日志异步批量写入优化

huangxiaoqiang

10 天以前 1ceaab54e82568a57599b61036aeccd110184e48

接口有效期控制与API日志异步批量写入优化

- 新增 App.ExpDateTime 字段，实现接口有效期控制，超期接口自动返回 500 错误
- 日志系统重构，采用 ConcurrentQueue 队列和 DataTable 批量写入日志，提升性能
- 新增 RequestLogModel 并通过依赖注入管理请求日志上下文
- API 日志中间件支持忽略指定接口（通过 ApiLogIgnore 配置），优化请求/响应参数采集
- 配置文件新增 ApiLogIgnore 字段，支持灵活配置日志忽略接口
- 注释掉 Quartz 任务中间件，暂不启用定时任务
- 增强异常处理，提升系统健壮性

 ÏîÄ¿´úÂë/WMS/WMSServer/WIDESEA_Core/App.cs

@@ -34,6 +34,11 @@
            set => _isRun = IsBuild = value;
        }

        /// <summary>
        /// è®¾ç½®æŽ¥å£æœ‰æ•ˆæˆªæ­¢æ—¶é—´ï¼Œè¶…过该时间将无法继续使用接口
        /// </summary>
        public static DateTime? ExpDateTime = null;

        /// <summary>应用有效程序集</summary>
        public static readonly IEnumerable<Assembly> Assemblies = RuntimeExtension.GetAllAssemblies();


 ÏîÄ¿´úÂë/WMS/WMSServer/WIDESEA_Core/Extensions/AutofacModuleRegister.cs

@@ -14,6 +14,7 @@
using WIDESEA_Core.BaseRepository;
using WIDESEA_Core.BaseServices;
using WIDESEA_Core.Helper;
using WIDESEAWCS_Core.LogHelper;

namespace WIDESEA_Core.Extensions
{
@@ -24,8 +25,8 @@
        {
            var cacheType = new List<Type>();

            builder.RegisterType<LogAOP>();
            cacheType.Add(typeof(LogAOP));
            //builder.RegisterType<LogAOP>();
            //cacheType.Add(typeof(LogAOP));

            builder.RegisterGeneric(typeof(RepositoryBase<>)).As(typeof(IRepository<>)).InstancePerDependency();//注册仓储
            builder.RegisterGeneric(typeof(ServiceBase<,>)).As(typeof(IService<>)).InstancePerDependency();//注册服务
@@ -73,6 +74,7 @@
               .AsImplementedInterfaces()
               .InstancePerLifetimeScope()
               .PropertiesAutowired();
            builder.RegisterType<RequestLogModel>().InstancePerLifetimeScope();
        }
    }
}

 ÏîÄ¿´úÂë/WMS/WMSServer/WIDESEA_Core/LogHelper/Logger.cs

@@ -2,7 +2,9 @@
using Microsoft.AspNetCore.Http;
using SqlSugar;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
@@ -10,115 +12,155 @@
using WIDESEA_Core.Helper;
using WIDESEA_Core.HttpContextUser;
using WIDESEA_Core.Seed;
using WIDESEAWCS_Core.LogHelper;

namespace WIDESEA_Core.LogHelper
{
    public class Logger
    {
        private static readonly ILog log = LogManager.GetLogger(typeof(Logger));

        public static void Debug(string message)
        public static ConcurrentQueue<dynamic> loggerQueueData = new ConcurrentQueue<dynamic>();
        static Logger()
        {
            Task.Run(() =>
            {
                StartWriteLog();
            });

        }

        public static void Debug(string message, Exception exception)
        static void StartWriteLog()
        {
            DataTable queueTable = CreateEmptyTable();
            while (true)
            {
                try
                {
                    if (loggerQueueData.Count() > 0 && queueTable.Rows.Count < 500)
                    {
                        DequeueToTable(queueTable); continue;
                    }
                    //每5秒写一次数据
                    Thread.Sleep(5000);

                    if (queueTable.Rows.Count == 0) { continue; }

                    SqlSugarScope sugarClient = new SqlSugarScope(new ConnectionConfig()
                    {
                        ConnectionString = DBContext.GetMainConnectionDb().Connection,
                        IsAutoCloseConnection = true,
                        DbType = MainDb.DbType,
                    });

                    int rows = sugarClient.Fastest<DataTable>().AS("Sys_Log").BulkCopy(queueTable);

                    queueTable.Clear();
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.ToString());
                }
            }
        }

        public static void Info(string message)
        private static void DequeueToTable(DataTable queueTable)
        {

            loggerQueueData.TryDequeue(out dynamic log);
            if (log != null)
            {
                DataRow row = queueTable.NewRow();
                if (log.BeginDate == null || log.BeginDate?.Year < 2010)
                {
                    log.BeginDate = DateTime.Now;
                }
                if (log.EndDate == null)
                {
                    log.EndDate = DateTime.Now;
                }
                //  row["Id"] = log.Id;
                row["RequestParam"] = log.RequestParam?.Replace("\r\n", "");
                row["ResponseParam"] = log.ResponseParam?.Replace("\r\n", "");
                //row["Success"] = log.Success ?? -1;
                row["BeginDate"] = log.BeginDate;
                row["EndDate"] = log.EndDate;
                row["ElapsedTime"] = ((DateTime)log.EndDate - (DateTime)log.BeginDate).TotalMilliseconds;
                row["UserIP"] = log.UserIP;
                row["Url"] = log.Url;
                row["UserId"] = log.UserId ?? -1;
                row["UserName"] = log.UserName;
                queueTable.Rows.Add(row);
            }
        }

        public static void Info(string message, Exception exception)
        private static DataTable CreateEmptyTable()
        {

            DataTable queueTable = new DataTable();
            queueTable.Columns.Add("BeginDate", Type.GetType("System.DateTime"));
            queueTable.Columns.Add("ElapsedTime", Type.GetType("System.Int32"));
            queueTable.Columns.Add("EndDate", Type.GetType("System.DateTime"));
            queueTable.Columns.Add("RequestParam", typeof(string));
            queueTable.Columns.Add("ResponseParam", typeof(string));
            //queueTable.Columns.Add("Success", Type.GetType("System.Int32"));
            queueTable.Columns.Add("Url", typeof(string));
            queueTable.Columns.Add("UserIP", typeof(string));
            queueTable.Columns.Add("UserName", typeof(string));
            queueTable.Columns.Add("UserId", Type.GetType("System.Int32"));
            //queueTable.Columns.Add("LogType", typeof(string));
            //queueTable.Columns.Add("ExceptionInfo", typeof(string));
            //queueTable.Columns.Add("ServiceIP", typeof(string));
            //queueTable.Columns.Add("BrowserType", typeof(string));
            //queueTable.Columns.Add("Role_Id", Type.GetType("System.Int32"));
            return queueTable;
        }

        public static void Warn(string message)
        public static void Add(string requestParameter, string responseParameter)
        {

        }

        public static void Warning(string message, Exception exception)
        {

        }

        public static void Error(string message)
        {

        }

        public static void Error(string message, Exception exception)
        {

        }

        public static void Fatal(string message)
        {

        }

        public static void Fatal(string message, Exception exception)
        {

        }

        public static void WriteApiLog2DB(HttpContext context, string requestParameter, DateTime beginDate, string responseParameter, DateTime endDate, LoggerStatus loggerStatus)
        {
            dynamic log = null;
            try
            {
                if (context.Request.Method == "OPTIONS") return;

                HttpContext context = App.HttpContext;
                if (context == null)
                {
                    Console.WriteLine($"未获取到httpcontext信息,reqParam:{requestParameter},respParam:{responseParameter},success:{loggerStatus}");
                    return;
                }
                if (context.Request.Method == "OPTIONS") return;
                RequestLogModel logModel = (context.RequestServices.GetService(typeof(RequestLogModel)) as RequestLogModel) ?? new RequestLogModel { RequestDate = DateTime.Now };

                Dictionary<string, object> dic = new Dictionary<string, object>
                IUser user = App.User;
                log = new
                {
                    {"BeginDate",beginDate },
                    {"ElapsedTime",(endDate - beginDate).TotalMilliseconds.DoubleToInt() },
                    {"EndDate",endDate },
                    {"RequestParam",requestParameter },
                    {"ResponseParam",responseParameter },
                    {"Success",1 },
                    {"Url",context.Request.Path.Value??"" },
                    {"UserIP",context.GetUserIp() },
                    {"UserName","App.User?.Name" },
                    {"User_Id","0" }

                    //{"BeginDate",beginDate },
                    //{"ElapsedTime",(endDate - beginDate).TotalMilliseconds.ObjToInt() },
                    //{"EndDate",endDate },
                    //{"RequestParam",requestParameter },
                    //{"ResponseParam",responseParameter },
                    //{"Success",1 },
                    //{"Url",context.Request.Path.Value??"" },
                    //{"UserIP",context.GetUserIp() },
                    //{"UserName","App.User?.Name" },
                    //{"User_Id","App.User?.ID" }
                    BeginDate = logModel.RequestDate,
                    EndDate = DateTime.Now,
                    RequestParam = requestParameter,
                    ResponseParam = responseParameter,
                    Url = context.Request.Scheme + "://" + context.Request.Host + context.Request.PathBase + context.Request.Path,
                    UserIP = GetClientIP(context)?.Replace("::ffff:", ""),
                    UserId = user.UserId,
                    UserName = user.UserName
                };


                SqlSugarClient sqlSugarClient = DBContext.GetCustomDB(DBContext.GetConnectionConfig());
                sqlSugarClient.Insertable(dic).AS("Sys_Log").ExecuteCommand();
            }
            catch (Exception ex)
            catch (Exception exception)
            {

                log = log ?? new
                {
                    BeginDate = DateTime.Now,
                    EndDate = DateTime.Now,
                    RequestParam = requestParameter,
                    ResponseParam = responseParameter,
                };
            }
            //添加系统日志
            loggerQueueData.Enqueue(log);
        }
    }

    public enum LoggerStatus
    {
        Success = 1,
        Error = 2,
        Info = 3
        public static string GetClientIP(HttpContext context)
        {
            var ip = context.Request.Headers["X-Forwarded-For"].ObjToString();
            if (string.IsNullOrEmpty(ip))
            {
                ip = context.Connection.RemoteIpAddress.ObjToString();
            }

            return ip;
        }
    }
}

 ÏîÄ¿´úÂë/WMS/WMSServer/WIDESEA_Core/LogHelper/RequestLogModel.cs

¶Ô±ÈÐÂÎļþ
@@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace WIDESEAWCS_Core.LogHelper
{
    public class RequestLogModel
    {
        public DateTime RequestDate {  get; set; }
    }
}

 ÏîÄ¿´úÂë/WMS/WMSServer/WIDESEA_Core/Middlewares/ApiLogMiddleware.cs

@@ -4,10 +4,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using WIDESEA_Core.Helper;
using WIDESEA_Core.LogHelper;
using WIDESEAWCS_Core.LogHelper;

namespace WIDESEA_Core.Middlewares
{
@@ -31,83 +33,101 @@
        //todo
        public async Task InvokeAsync(HttpContext context)
        {
            //if (AppSettings.app("Middleware", "RequestResponseLog", "Enabled").ObjToBool())
            if (App.ExpDateTime != null && (DateTime.Now - App.ExpDateTime.GetValueOrDefault()).TotalSeconds > 0)
            {
                // è¿‡æ»¤ï¼Œåªæœ‰æŽ¥å£
                if (context.Request.Path.Value.Contains("api"))
                {
                    context.Request.EnableBuffering();
                    //Stream originalBody = context.Response.Body;
                context.Response.StatusCode = HttpStatusCode.InternalServerError.ObjToInt();
                context.Response.ContentType = "application/json";

                var json = new WebResponseContent();

                json.Message = HttpStatusCode.InternalServerError.ToString();//错误信息
                json.Code = 500;//500异常 

                StreamWriter streamWriter = new StreamWriter(context.Response.Body);
                await streamWriter.WriteAsync(json.Serialize());
                return;
            }

            // è¿‡æ»¤ï¼Œåªæœ‰æŽ¥å£
            if (context.Request.Path.Value?.Contains("api") ?? false)
            {
                context.Request.EnableBuffering();
                Stream originalBody = context.Response.Body;
                string requestParam = string.Empty;
                string responseParam = string.Empty;
                try
                {
                    string? apiIgnore = AppSettings.GetValue("ApiLogIgnore")?.ToString();
                    string[] ignoreUrls = !string.IsNullOrEmpty(apiIgnore) ? apiIgnore.Split(",") : new string[] { "get" };

                    (context.RequestServices.GetService(typeof(RequestLogModel)) as RequestLogModel).RequestDate = DateTime.Now;
                    try
                    {
                        // å­˜å‚¨è¯·æ±‚数据
                        //string requestParam = GetRequestData(context);
                        //DateTime beginDate = DateTime.Now;

                        //using var ms = new MemoryStream();
                        //context.Response.Body = ms;

                        await _next(context);

                        // å­˜å‚¨å“åº”数据
                        //DateTime endDate = DateTime.Now;
                        //string responseParam = GetResponsetData(context);

                        //context.Response.Body.Position = 0;
                        //await context.Response.Body.CopyToAsync(originalBody);

                        //Logger.WriteApiLog2DB(context,requestParam, beginDate, responseParam, endDate, context.Response.StatusCode == 200 ? LoggerStatus.Success : LoggerStatus.Error);
                        requestParam = RequestDataLog(context);
                        context.Request.Body.Position = 0;
                    }
                    catch (Exception ex)
                    {
                        // è®°å½•å¼‚常                        
                        _logger.LogError(ex.Message + "" + ex.InnerException);
                    }
                    finally
                    {
                        //context.Response.Body = originalBody;
                    }
                }
                else
                {
                    catch { }
                    using MemoryStream ms = new();
                    context.Response.Body = ms;

                    await _next(context);

                    try
                    {
                        // å­˜å‚¨å“åº”数据
                        responseParam = ResponseDataLog(context.Response);
                    }
                    catch { }

                    ms.Position = 0;
                    await ms.CopyToAsync(originalBody);

                    if (!ignoreUrls.Any(x => context.Request.Path.Value?.ToLower().Contains(x.ToLower()) ?? false))
                    {
                        Logger.Add(requestParam, responseParam);
                    }
                }
                catch (Exception ex)
                {
                    // è®°å½•å¼‚常

                }
                finally
                {
                    context.Response.Body = originalBody;
                }
            }
            //else
            //{
            //    await _next(context);
            //}
        }

        private string GetRequestData(HttpContext context)
        {
            try
            else
            {
                using StreamReader sr = new StreamReader(context.Request.Body);
                string request = JsonConvert.SerializeObject(sr.ReadToEnd()); ;
                context.Request.Body.Position = 0;
                return request;
            }
            catch (Exception ex)
            {
                return $"请求参数获取错误,{ex.Message}";
                await _next(context);
            }
        }

        private string GetResponsetData(HttpContext context)
        private string RequestDataLog(HttpContext context)
        {
            try
            var request = context.Request;
            var sr = new StreamReader(request.Body);

            object obj = new
            {
                using StreamReader sr = new StreamReader(context.Response.Body);
                string response = JsonConvert.SerializeObject(sr.ReadToEnd()); ;
                context.Response.Body.Position = 0;
                return response;
            }
            catch (Exception ex)
            {
                return $"响应参数获取错误,{ex.Message}";
            }
                QueryString = request.QueryString.ToString(),
                BodyData = sr.ReadToEndAsync().Result
            };

            string data = JsonConvert.SerializeObject(obj);

            request.Body.Position = 0;

            return data;
        }

        private string ResponseDataLog(HttpResponse response)
        {
            response.Body.Position = 0;
            using StreamReader stream = new StreamReader(response.Body, leaveOpen: true);
            string body = stream.ReadToEnd();
            response.Body.Position = 0;
            return body;
        }
    }
}

 ÏîÄ¿´úÂë/WMS/WMSServer/WIDESEA_WMSServer/Program.cs

@@ -121,7 +121,7 @@

var app = builder.Build();

app.UseQuartzJobMildd();
//app.UseQuartzJobMildd();
// 3、配置中间件
app.UseMiniProfiler();//性能分析器
app.ConfigureApplication();//配置文件

 ÏîÄ¿´úÂë/WMS/WMSServer/WIDESEA_WMSServer/appsettings.json

@@ -23,7 +23,8 @@
  },
  "ApiName": "WIDESEA",
  "ExpMinutes": 120,

  //过滤接口
  "ApiLogIgnore": "get,query,download,upload,template",
  // éœ€è¦ç§»åº“çš„è¡Œ
  "TransfertRows": "1,4,5,8"
}