//============================================================== // Copyright (C) 2020 Inc. All rights reserved. // //============================================================== // Create by 种道洋 at 2020/4/16 13:33:36. // Version 1.0 // 种道洋 //============================================================== using System; using System.Collections.Generic; using System.Net.Http.Headers; using System.Numerics; using System.Text; namespace Cdy.Tag { /* * ****文件结构**** * 一个文件头 + 多个数据区组成 , 一个数据区:数据区头+数据块指针区+数据块区 * [] 表示重复的一个或多个内容 * HisData File Structor FileHead(84) + [HisDataRegion] FileHead: dataTime(8)(FileTime)+dataTime(8)(LastUpdateTime)+DataRegionCount(4)+DatabaseName(64) HisDataRegion Structor: RegionHead + DataBlockPoint Area + DataBlock Area RegionHead: PreDataRegionPoint(8) + NextDataRegionPoint(8) + Datatime(8)+ tagcount(4)+ tagid sum(8)+file duration(4)+block duration(4)+Time tick duration(4) DataBlockPoint Area: [ID]+[block Point] [block point]: [[tag1 point,tag2 point,....][tag1 point,tag2 point,...].....] 以时间单位对变量的数去区指针进行组织 DataBlock Area: [block size + data block] */ /// /// /// public class DataFileInfo { #region ... Variables ... private SortedDictionary> mTimeOffsets = new SortedDictionary>(); private bool mInited = false; private static object mLockObj = new object(); private DateTime mLastTime; private long mLastProcessOffset = -1; /// /// /// private int mRegionCount = 0; #endregion ...Variables... #region ... Events ... #endregion ...Events... #region ... Constructor... #endregion ...Constructor... #region ... Properties ... /// /// /// public DateTime LastTime { get { return mLastTime; } } /// /// /// public string FId { get; set; } /// /// /// public string FileName { get; set; } /// /// 开始时间 /// public DateTime StartTime { get; set; } /// /// /// public DateTime EndTime { get { return StartTime + Duration; } set { Duration = value - StartTime; } } /// /// 时间长度 /// public TimeSpan Duration { get; set; } #endregion ...Properties... #region ... Methods ... /// /// /// public void UpdateLastDatetime() { lock (mLockObj) { mTimeOffsets.Clear(); Scan(); if (DataFileManager.CurrentDateTime.ContainsKey(FId)) { DataFileManager.CurrentDateTime[FId] = mLastTime; } else { DataFileManager.CurrentDateTime.Add(FId, mLastTime); } } } /// /// /// /// public void Scan() { long offset = DataFileManager.FileHeadSize; DateTime time, tmp; using (var ss = DataFileSeriserManager.manager.GetDefaultFileSersie()) { ss.OpenForReadOnly(FileName); //读取文件时间 DateTime fileTime = ss.ReadDateTime(0); //读取最后一次更新时间 mLastTime = ss.ReadDateTime(8); var rcount = ss.ReadInt(16); if(rcount!= mRegionCount) { mRegionCount = rcount; } else { return; } do { //读取数据区时间 time = ss.ReadDateTime(offset + 16); long oset = offset; //读取下个区域位置 offset = ss.ReadLong(offset + 8); if (offset != 0) { var dt2 = ss.ReadDateTime(offset + 16); mTimeOffsets.Add(time, new Tuple(dt2 - time, oset, dt2)); tmp = dt2; } else { var tspan = StartTime + Duration - time; if (tspan.TotalMilliseconds > 0) mTimeOffsets.Add(time, new Tuple(tspan, oset, time + tspan)); tmp = time + tspan; } mLastProcessOffset = oset; } while (offset != 0); if (mLastTime <= fileTime) { mLastTime = tmp; } //if (mLastProcessOffset == -1) //{ //} //else //{ // offset = mLastProcessOffset; // //读取数据区时间 // time = ss.ReadDateTime(offset + 16); // long oset = offset; // //读取下个区域位置 // offset = ss.ReadLong(offset + 8); // if (offset != 0) // { // var dt2 = ss.ReadDateTime(offset + 16); // if (mTimeOffsets.ContainsKey(time)) // { // mTimeOffsets[time] = new Tuple(dt2 - time, oset, dt2); // } // else // { // mTimeOffsets.Add(time, new Tuple(dt2 - time, oset, dt2)); // } // tmp = dt2; // } // else // { // var tspan = StartTime + Duration - time; // if (tspan.TotalMilliseconds > 0) // { // if (mTimeOffsets.ContainsKey(time)) // { // mTimeOffsets[time] = new Tuple(tspan, oset, time + tspan); // } // else // { // mTimeOffsets.Add(time, new Tuple(tspan, oset, time + tspan)); // } // } // tmp = time + tspan; // } //} mInited = true; } } /// /// /// /// /// public long GetFileOffsets(DateTime time) { lock (mLockObj) if (!mInited) Scan(); foreach (var vv in mTimeOffsets) { if (vv.Key <= time && time < (vv.Key + vv.Value.Item1)) { return vv.Value.Item2; } } return -1; } /// /// /// /// /// /// public Dictionary> GetFileOffsets(DateTime startTime, DateTime endTime) { lock (mLockObj) if (!mInited) Scan(); Dictionary> re = new Dictionary>(); foreach (var vv in mTimeOffsets) { if (vv.Key >= startTime && vv.Key < endTime) { re.Add(vv.Key, vv.Value); } } return re; } /// /// 获取某个时间段内,不包括数据的时间段集合 /// /// /// /// public List> GetNoValueOffsets(DateTime startTime, DateTime endTime) { List> re = new List>(); DateTime stime = startTime; List dtmp = new List(); DateTimeSpan ddt = new DateTimeSpan() { Start = startTime, End = endTime }; foreach (var vv in mTimeOffsets) { DateTimeSpan dts = new DateTimeSpan() { Start = vv.Key, End = vv.Value.Item3 }; dts = dts.Cross(ddt); if (!dts.IsEmpty()) dtmp.Add(dts); } foreach (var vv in dtmp) { if (vv.Start > stime) { re.Add(new Tuple(stime, vv.Start)); } stime = vv.End; } return re; } /// /// 判断某个时间点是否有数据 /// /// /// public bool HasValue(DateTime time) { foreach (var vv in mTimeOffsets) { if (vv.Key <= time && time < vv.Value.Item3) { return true; } } return false; } #endregion ...Methods... #region ... Interfaces ... #endregion ...Interfaces... } /// /// /// public class DateTimeSpan { /// /// /// public DateTimeSpan Empty = new DateTimeSpan() { Start = DateTime.MinValue, End = DateTime.MinValue }; /// /// /// public DateTime Start { get; set; } /// /// /// public DateTime End { get; set; } /// /// /// /// public bool IsEmpty() { return Start == DateTime.MinValue && End == DateTime.MinValue; } /// /// /// /// public bool IsZore() { return (End - Start).TotalSeconds == 0; } /// /// /// /// /// public DateTimeSpan Cross(DateTimeSpan target) { DateTime stime = Max(target.Start, this.Start); DateTime etime = Min(target.End, this.End); if (etime < stime) { return Empty; } else { return new DateTimeSpan() { Start = stime, End = etime }; } } /// /// /// /// /// /// public DateTime Min(DateTime time1, DateTime time2) { return time1 <= time2 ? time1 : time2; } /// /// /// /// /// /// public DateTime Max(DateTime time1, DateTime time2) { return time1 >= time2 ? time1 : time2; } } public static class DataFileInfoExtend { /// /// /// /// /// public static DataFileSeriserbase GetFileSeriser(this DataFileInfo file) { var re = DataFileSeriserManager.manager.GetDefaultFileSersie(); re.FileName = file.FileName; re.OpenForReadOnly(file.FileName); return re; } #region 读取所有值 /// /// 读取某时间段内的所有bool值 /// /// /// /// /// /// public static void ReadAllValue(this DataFileInfo file, int tid, DateTime startTime, DateTime endTime, HisQueryResult result) { var vff = file.GetFileSeriser(); //Dictionary> moffs = new Dictionary>(); var offset = file.GetFileOffsets(startTime, endTime); foreach (var vv in offset) { DateTime stime = vv.Key > startTime ? vv.Key : startTime; DateTime etime = vv.Key + vv.Value.Item1 > endTime ? endTime : vv.Key + vv.Value.Item1; vff.ReadAllValue(vv.Value.Item2, tid, stime, etime, result); } vff.Close(); //GeneratorTime(moffs, startTime, endTime, file); //foreach (var vf in moffs) //{ // vff.ReadAllValue(vf.Key, tid, vf.Value.Item1, vf.Value.Item2, result); //} } #endregion #region 读取指定时刻值 /// /// /// /// /// /// /// /// /// public static object Read(this DataFileInfo file, int tid, DateTime time, QueryValueMatchType type) { using (var vff = file.GetFileSeriser()) { var offset = file.GetFileOffsets(time); return vff.Read(offset, tid, time, type); } } /// /// /// /// /// /// /// /// /// public static HisQueryResult Read(this DataFileInfo file, int tid, List times, QueryValueMatchType type) { HisQueryResult re = new HisQueryResult(times.Count); Read(file, tid, times, type, re); return re; } /// /// /// /// /// /// /// /// /// public static void Read(this DataFileInfo file, int tid, List times, QueryValueMatchType type, HisQueryResult result) { using (var vff = file.GetFileSeriser()) { Dictionary> moffs = new Dictionary>(); foreach (var vv in times) { var ff = file.GetFileOffsets(vv); if (moffs.ContainsKey(ff)) { moffs[ff].Add(vv); } else { moffs.Add(ff, new List() { vv }); } } foreach (var vf in moffs) { if (vf.Key > -1) vff.Read(vf.Key, tid, vf.Value, type, result); else { foreach (var vv in vf.Value) { result.Add(default(T), vv, (byte)QualityConst.Null); } } } } } #endregion #region DataFileSeriser Read /// /// /// /// /// /// /// /// /// /// public static void Read(this DataFileSeriserbase datafile, long offset, int tid, List dataTimes, QueryValueMatchType type, HisQueryResult res) { int timetick = 0; var data = datafile.ReadTagDataBlock2(tid, offset, dataTimes, out timetick); foreach (var vv in data) { DeCompressDataBlockValue(vv.Key, vv.Value, timetick, type, res); } foreach (var vv in data) { vv.Key.Dispose(); } data.Clear(); } /// /// /// /// /// /// /// /// /// /// public static object Read(this DataFileSeriserbase datafile, long offset, int tid, DateTime dataTime, QueryValueMatchType type) { int timetick = 0; using (var data = datafile.ReadTagDataBlock(tid, offset, dataTime, out timetick)) { return DeCompressDataBlockValue(data, dataTime, timetick, type); } } /// /// /// /// /// /// /// /// /// public static void ReadAllValue(this DataFileSeriserbase datafile, long offset, int tid, DateTime startTime, DateTime endTime, HisQueryResult result) { int timetick = 0; var data = datafile.ReadTagDataBlock2(tid, offset, startTime, endTime, out timetick); foreach (var vv in data) { DeCompressDataBlockAllValue(vv.Key, vv.Value.Item1, vv.Value.Item2, timetick, result); } // System.Threading.Tasks.Parallel.ForEach(data, (vv) => { DeCompressDataBlockAllValue(vv.Key, vv.Value.Item1, vv.Value.Item2, timetick, result); }); foreach (var vv in data) { vv.Key.Dispose(); } data.Clear(); } #endregion #region DeCompressData /// /// /// /// /// /// /// /// /// private static object DeCompressDataBlockValue(MarshalMemoryBlock memory, DateTime datatime, int timeTick, QueryValueMatchType type) { //MarshalMemoryBlock target = new MarshalMemoryBlock(memory.Length); //读取压缩类型 var ctype = memory.ReadByte(); var tp = CompressUnitManager2.Manager.GetCompress(ctype); if (tp != null) { return tp.DeCompressValue(memory, 1, datatime, timeTick, type); } return null; } /// /// /// /// /// /// /// /// /// private static void DeCompressDataBlockValue(MarshalMemoryBlock memory, List datatime, int timeTick, QueryValueMatchType type, HisQueryResult result) { //MarshalMemoryBlock target = new MarshalMemoryBlock(memory.Length); //读取压缩类型 var ctype = memory.ReadByte(); var tp = CompressUnitManager2.Manager.GetCompress(ctype); if (tp != null) { tp.DeCompressValue(memory, 1, datatime, timeTick, type, result); } } /// /// /// /// /// /// /// /// private static void DeCompressDataBlockAllValue(MarshalMemoryBlock memory, DateTime startTime, DateTime endTime, int timeTick, HisQueryResult result) { //MarshalMemoryBlock target = new MarshalMemoryBlock(memory.Length); //读取压缩类型 var ctype = memory.ReadByte(); var tp = CompressUnitManager2.Manager.GetCompress(ctype); if (tp != null) { tp.DeCompressAllValue(memory, 1, startTime, endTime, timeTick, result); } } #endregion #region 读取数据区域头数据 /// /// 检测数据头部指针区域数据是否被缓存 /// /// /// /// /// /// /// public static Dictionary CheckBlockHeadCach(this DataFileSeriserbase datafile, long offset, out int tagCount, out int fileDuration, out int blockDuration, out int timetick, out long blockPointer, out DateTime time) { //文件头部结构:Pre DataRegion(8) + Next DataRegion(8) + Datatime(8)+tagcount(4)+ tagid sum(8) +file duration(4)+ block duration(4)+Time tick duration(4)+ { + tagid1+tagid2+...+tagidn }+ {[tag1 block point1(8) + tag2 block point1+ tag3 block point1+...] + [tag1 block point2(8) + tag2 block point2+ tag3 block point2+...]....} var dataoffset = offset + 16; //读取时间 time = datafile.ReadDateTime(dataoffset); dataoffset += 8; //读取变量个数 int count = datafile.ReadInt(dataoffset); dataoffset += 4; tagCount = count; //读取校验和 long idsum = datafile.ReadLong(dataoffset); dataoffset += 8; //读取单个文件的时长 fileDuration = datafile.ReadInt(dataoffset); dataoffset += 4; //读取数据块时长 blockDuration = datafile.ReadInt(dataoffset); dataoffset += 4; //读取时钟周期 timetick = datafile.ReadInt(dataoffset); dataoffset += 4; lock (TagHeadOffsetManager.manager) { if (!TagHeadOffsetManager.manager.Contains(idsum, count)) { //Tag id 列表经过压缩,内容格式为:DataSize + Data var dsize = datafile.ReadInt(dataoffset); if (dsize <= 0) { tagCount = 0; fileDuration = 0; blockDuration = 0; timetick = 0; blockPointer = 0; return new Dictionary(); } dataoffset += 4; blockPointer = dataoffset + dsize - offset; var dtmp = new Dictionary(); using (var dd = datafile.Read(dataoffset, dsize)) { MarshalVarintCodeMemory vcm = new MarshalVarintCodeMemory(dd.StartMemory, dsize); var ltmp = vcm.ToIntList(); //vcm.Dispose(); if (ltmp.Count > 0) { int preid = ltmp[0]; dtmp.Add(preid, 0); for (int i = 1; i < ltmp.Count; i++) { var id = ltmp[i] + preid; dtmp.Add(id, i); preid = id; } } TagHeadOffsetManager.manager.Add(idsum, count, dtmp, blockPointer); } return dtmp; } else { var re = TagHeadOffsetManager.manager.Get(idsum, count); blockPointer = re.Item2; return re.Item1; } } } /// /// 读取某个变量在头部文件种的序号 /// /// /// /// /// /// /// /// public static int ReadTagIndexInDataPointer(this DataFileSeriserbase datafile, int tid, long offset, out int tagCount, out int fileDuration, out int blockDuration, out int timetick, out long blockpointer, out DateTime time) { var hfile = datafile.CheckBlockHeadCach(offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time); if (hfile.ContainsKey(tid)) { return hfile[tid]; } return -1; } /// /// 读取数据区域的数据头数据 /// /// /// /// /// /// /// /// public static List ReadTargetBlockAddress(this DataFileSeriserbase datafile, List tid, long offset, out int tagCount, out int fileDuration, out int blockDuration, out int timetick, out long blockpointer, out DateTime time) { var hfile = datafile.CheckBlockHeadCach(offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time); List re = new List(); foreach (var vv in tid) { if (hfile.ContainsKey(vv)) { re.Add(hfile[vv]); } else { re.Add(-1); } } return re; } #endregion #region 读取数据块 /// /// /// /// /// /// /// /// /// public static MarshalMemoryBlock ReadTagDataBlock(this DataFileSeriserbase datafile, int tid, long offset, DateTime dataTime, out int timetick) { int fileDuration, blockDuration = 0; int tagCount = 0; DateTime time; long blockpointer = 0; var blockIndex = datafile.ReadTagIndexInDataPointer(tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time); int blockcount = fileDuration * 60 / blockDuration; var startTime = datafile.ReadDateTime(16); var ttmp = (dataTime - startTime).TotalMinutes; int dindex = (int)(ttmp / blockDuration); if (ttmp % blockDuration > 0) { dindex++; } if (dindex > blockcount) { throw new Exception("DataPointer index is out of total block number"); } var dataPointer = datafile.ReadLong(blockIndex * 8 + dindex * tagCount * 8); //读取DataBlock的地址 var datasize = datafile.ReadInt(dataPointer); //读取DataBlock 的大小 return datafile.Read(dataPointer + 4, datasize); } /// /// /// /// /// /// /// /// /// public static Dictionary ReadTagDataBlock(this DataFileSeriserbase datafile, int tid, long offset, List dataTimes, out int timetick) { int fileDuration, blockDuration = 0; int tagCount = 0; long blockpointer = 0; DateTime time; var blockIndex = datafile.ReadTagIndexInDataPointer(tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time); int blockcount = fileDuration * 60 / blockDuration; var startTime = datafile.ReadDateTime(16); Dictionary rtmp = new Dictionary(); Dictionary re = new Dictionary(); foreach (var vdd in dataTimes) { var ttmp = (vdd - startTime).TotalMinutes; int dindex = (int)(ttmp / blockDuration); if (ttmp % blockDuration > 0) { dindex++; } if (dindex > blockcount) { throw new Exception("DataPointer index is out of total block number"); } var dataPointer = datafile.ReadLong(blockIndex * 8 + dindex * tagCount * 8); //读取DataBlock的地址 var datasize = datafile.ReadInt(dataPointer); //读取DataBlock 的大小 if (!rtmp.ContainsKey(dataPointer)) { var rmm = datafile.Read(dataPointer + 4, datasize); re.Add(vdd, rmm); rtmp.Add(dataPointer, rmm); } else { re.Add(vdd, rtmp[dataPointer]); } } return re; } /// /// /// /// /// /// /// /// /// public static Dictionary> ReadTagDataBlock2(this DataFileSeriserbase datafile, int tid, long offset, List dataTimes, out int timetick) { int fileDuration, blockDuration = 0; int tagCount = 0; long blockpointer = 0; DateTime time; var tagIndex = datafile.ReadTagIndexInDataPointer(tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time); Dictionary rtmp = new Dictionary(); Dictionary> re = new Dictionary>(); if (tagCount == 0) return re; int blockcount = fileDuration * 60 / blockDuration; var startTime = datafile.ReadDateTime(0); //var startTime = time; foreach (var vdd in dataTimes) { var ttmp = (vdd - startTime).TotalMinutes; int blockindex = (int)(ttmp / blockDuration); //if (ttmp % blockDuration > 0) //{ // blockindex++; //} if (blockindex > blockcount) { throw new Exception("DataPointer index is out of total block number"); } var dataPointer = datafile.ReadLong(offset + blockpointer + tagIndex * 8 + blockindex * tagCount * 8); //读取DataBlock的地址 if (dataPointer > 0) { var datasize = datafile.ReadInt(dataPointer); //读取DataBlock 的大小 if (datasize > 0) { //var rmm = datafile.Read(dataPointer + 4, (int)datasize); //if (!re.ContainsKey(rmm)) //{ // re.Add(rmm, new Tuple(sstart, end)); //} if (!rtmp.ContainsKey(dataPointer)) { var rmm = datafile.Read(dataPointer + 4, datasize); if (!re.ContainsKey(rmm)) { re.Add(rmm, new List() { vdd }); } else { re[rmm].Add(vdd); } rtmp.Add(dataPointer, rmm); } else { var rmm = rtmp[dataPointer]; if (!re.ContainsKey(rmm)) { re.Add(rmm, new List() { vdd }); } else { re[rmm].Add(vdd); } } } } } return re; } /// /// /// /// /// /// /// /// /// /// /// public static Dictionary> ReadTagDataBlock2(this DataFileSeriserbase datafile, int tid, long offset, DateTime start, DateTime end, out int timetick) { int fileDuration, blockDuration = 0; int tagCount = 0; long blockpointer = 0; DateTime time; var tagIndex = datafile.ReadTagIndexInDataPointer(tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time); int blockcount = fileDuration * 60 / blockDuration; //读取文件开始时间 var startTime = datafile.ReadDateTime(0); //var startTime = time; Dictionary> re = new Dictionary>(); DateTime sstart = start; DateTime send = end; while (sstart < end) { var ttmp = Math.Round((sstart - startTime).TotalSeconds, 3); var vv = blockDuration * 60 - (ttmp % (blockDuration * 60)); send = sstart.AddSeconds(vv); //send = (sstart - new TimeSpan(0, 0, 0, sstart.Second, sstart.Millisecond)).AddMinutes(blockDuration); if (send > end) { send = end; } int blockindex = (int)(ttmp / (blockDuration * 60)); //if (ttmp % blockDuration > 0) //{ // dindex++; //} if (blockindex > blockcount) { throw new Exception("DataPointer index is out of total block number"); } var dataPointer = datafile.ReadLong(offset + blockpointer + tagIndex * 8 + blockindex * tagCount * 8); //读取DataBlock的地址 if (dataPointer > 0) { var datasize = datafile.ReadInt(dataPointer); //读取DataBlock 的大小 if (datasize > 0) { var rmm = datafile.Read(dataPointer + 4, (int)datasize); if (!re.ContainsKey(rmm)) { re.Add(rmm, new Tuple(sstart, send)); } } } sstart = send; } return re; } #endregion } }