1.需求:
計算滑動時間下的1小時、3小時、6小時、12小時、24小時降水數據,統計這個時間下的分鐘級降水數據
2.分析
第一版本:降水分鐘級數據保存時間不長,保存太多意義不大,以更新的形式來保存這些統計數據效果會比較好,使用管道先進先出的原則每5分鐘取一次數據,每次只需要取近5分鐘的數據即可,設置管道長度分別為12組、36組、72組,144組和288組降水數據,取夠數據之后將之前第一組的數據自動去除即可
版本分析:
優點:基本能達到在一次過程中的數據統計功能,而且占用內存很小,數據整體運行比較順暢,每次取的數據量比較小
缺點:數據統計不準確,雖然用了定時器的定數回調方法來處理數據,仍然存在數據統計不準的情況,這個問題初步判定為天擎數據的后期更新,但是管道內的數據沒有更新導致數據存在差異的問題,數據的準確性不能保證,棄用
第二個版本:保留每5分鐘更新一次數據的頻率,將近1小時、3小時、6小時、12小時、24小時降水數據直接進行統計,統計后的數據保存下來即可
版本分析:
優點:數據的準確性能保證,數據沒有差異
缺點:數據的體量比較大,內存占用比較大,在統計過程中存在內存占用很高的情況
版本更新:
在每次定時回調統計數據的方法中將每次調用后的法中對已經棄用的一些變量和占用內存較高的變量進行GC處理
更新分析:
優點:數據統計的內存占比在每次統計完成后會清理掉,占用的內存會比較小
缺點:每次都要統計很多數據,有些站點的數據在處理的時候會在時間段內漏報,在統計時會出現站點報錯,導致站點數據統計不上的情況,在統計的時候加上判斷
最終效果:
using System.Data;
using System.Data.SqlClient;
using System.Text;
using CMISS_DATA_Helper;
using Model;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NPOI.SS.Formula.Functions;class Program
{private static readonly Dictionary<string, decimal> hourlyAccumulator_1 = new Dictionary<string, decimal>();private static readonly Dictionary<string, decimal> hourlyAccumulator_3 = new Dictionary<string, decimal>();private static readonly Dictionary<string, decimal> hourlyAccumulator_6 = new Dictionary<string, decimal>();private static readonly Dictionary<string, decimal> hourlyAccumulator_12 = new Dictionary<string, decimal>();private static readonly Dictionary<string, decimal> hourlyAccumulator_24 = new Dictionary<string, decimal>();private static readonly object lockObj = new object();private static Timer timer;private static string DbConnectionString = "Data Source=.;Initial Catalog=Meteorological_Bz;User Id=sa;Password=sa123...;MultipleActiveResultSets=True";// 配置參數private static int IntervalMinutes = 5;private static int WindowSize_1 = (60 / IntervalMinutes) * 1; // 1小時 = 12 * 5分鐘private static int WindowSize_3 = (60 / IntervalMinutes) * 3;private static int WindowSize_6 = (60 / IntervalMinutes) * 6;private static int WindowSize_12 = (60 / IntervalMinutes) * 12;private static int WindowSize_24 = (60 / IntervalMinutes) * 24;private static string columns = "Station_Id_C,UPDATE_TIME,Datetime,PRE";private static string url = "";//天擎接口private static string AdminCode = "110000";//區劃碼private static string[] stationIds = getStations();//獲取站點數據
static JObject valueconfig;
static async Task Main(string[] args)
{// 初始化站點數據(假設已知400個站點ID)System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);using (System.IO.StreamReader file = new System.IO.StreamReader(AppDomain.CurrentDomain.BaseDirectory + "dataconfig.json", Encoding.UTF8)){valueconfig = JsonConvert.DeserializeObject<JObject>(file.ReadToEnd());file.Close();}DbConnectionString = valueconfig["conn"].ToString();if (valueconfig["stationIds"].ToString() != string.Empty) stationIds = valueconfig["stationIds"].ToString().Split(",");columns = valueconfig["columns"].ToString();url = valueconfig["url"].ToString();AdminCode = valueconfig["AdminCode"].ToString();IntervalMinutes = int.Parse(valueconfig["IntervalMinutes"].ToString());InitializeAccumulator(stationIds);// 初始化定時器(立即啟動,每5分鐘執行)timer = new Timer(ProcessWeatherData, null, 0, IntervalMinutes * 60 * 1000);Console.WriteLine("服務已啟動,按任意鍵退出...");Console.ReadKey();timer.Dispose();
}
// 初始化累加器(所有站點歸零)
private static void InitializeAccumulator(string[] stationIds)
{lock (lockObj){Console.WriteLine("初始化站點!");foreach (var id in stationIds){hourlyAccumulator_1[id] = 0;hourlyAccumulator_3[id] = 0;hourlyAccumulator_6[id] = 0;hourlyAccumulator_12[id] = 0;hourlyAccumulator_24[id] = 0;}}
}// 定時器回調方法private static async void ProcessWeatherData(object state){try{InitializeAccumulator(stationIds);// 1. 獲取API數據var currentData_1 = await FetchWeatherDataAsync(60 * 1);var currentData_3 = await FetchWeatherDataAsync(60 * 3);var currentData_6 = await FetchWeatherDataAsync(60 * 6);var currentData_12 = await FetchWeatherDataAsync(60 * 12);var currentData_24 = await FetchWeatherDataAsync(60 * 24);lock (lockObj){try{foreach (var kvp in currentData_1){if (hourlyAccumulator_1.ContainsKey(kvp.Key)){hourlyAccumulator_1[kvp.Key] += kvp.Value;}else{hourlyAccumulator_1.Add(kvp.Key, kvp.Value);}}//Console.WriteLine($"{DateTime.Now}: 1小時降水已統計");}catch (Exception ex){Console.WriteLine($"1小時降水已統計異常{ex.Message}");}try{foreach (var kvp in currentData_3){if (hourlyAccumulator_3.ContainsKey(kvp.Key)){hourlyAccumulator_3[kvp.Key] += kvp.Value;}else{hourlyAccumulator_3.Add(kvp.Key, kvp.Value);}}}catch (Exception ex){Console.WriteLine($"3小時降水已統計異常{ex.Message}");}try{foreach (var kvp in currentData_6){if (hourlyAccumulator_6.ContainsKey(kvp.Key)){hourlyAccumulator_6[kvp.Key] += kvp.Value;}else{hourlyAccumulator_6.Add(kvp.Key, kvp.Value);}}}catch (Exception ex){Console.WriteLine($"6小時降水已統計異常{ex.Message}");}try{foreach (var kvp in currentData_12){if (hourlyAccumulator_12.ContainsKey(kvp.Key))hourlyAccumulator_12[kvp.Key] += kvp.Value;else{hourlyAccumulator_12.Add(kvp.Key, kvp.Value);}}}catch (Exception ex){Console.WriteLine($"12小時降水已統計異常{ex.Message}");}try{foreach (var kvp in currentData_24){if (hourlyAccumulator_24.ContainsKey(kvp.Key))hourlyAccumulator_24[kvp.Key] += kvp.Value;else{hourlyAccumulator_24.Add(kvp.Key, kvp.Value);}}}catch (Exception ex){Console.WriteLine($"24小時降水已統計異常{ex.Message}");}// 4. 更新數據庫UpdateDatabase(hourlyAccumulator_1, hourlyAccumulator_3, hourlyAccumulator_6, hourlyAccumulator_12, hourlyAccumulator_24);Console.WriteLine($"{DateTime.Now}:近小時數據已更新");//5.釋放內存try{Console.WriteLine($"{DateTime.Now}:釋放內存");currentData_1.Clear();currentData_1.TrimExcess();currentData_1 = null;currentData_3.Clear();currentData_3.TrimExcess();currentData_3 = null;currentData_6.Clear();currentData_6.TrimExcess();currentData_6 = null;currentData_12.Clear();currentData_12.TrimExcess();currentData_12 = null;currentData_24.Clear();currentData_24.TrimExcess();currentData_24 = null;hourlyAccumulator_1.Clear();hourlyAccumulator_1.TrimExcess();hourlyAccumulator_3.Clear();hourlyAccumulator_3.TrimExcess();hourlyAccumulator_6.Clear();hourlyAccumulator_6.TrimExcess();hourlyAccumulator_12.Clear();hourlyAccumulator_12.TrimExcess();hourlyAccumulator_24.Clear();hourlyAccumulator_24.TrimExcess();GC.Collect();GC.WaitForPendingFinalizers();GC.Collect();}catch (Exception ex){Console.WriteLine(ex.ToString());}}}catch (Exception ex){Console.WriteLine($"處理失敗: {ex.Message}");}}private static async Task<Dictionary<string, decimal>> FetchWeatherDataAsync(int minute){
//獲取天擎數據,并異步返回Dictionary數據return data;
}private static void UpdateDatabase(Dictionary<string, decimal> data_1, Dictionary<string, decimal> data_3, Dictionary<string, decimal> data_6, Dictionary<string, decimal> data_12, Dictionary<string, decimal> data_24){
//更新數據庫
}
}