//==============================================================
// Copyright (C) 2020 Inc. All rights reserved.
//
//==============================================================
// Create by 种道洋 at 2020/4/16 13:33:36.
// Version 1.0
// 种道洋
//==============================================================
using System;
using System.Collections.Generic;
using System.Net.Http.Headers;
using System.Numerics;
using System.Text;
namespace Cdy.Tag
{
/*
* ****文件结构****
* 一个文件头 + 多个数据区组成 , 一个数据区:数据区头+数据块指针区+数据块区
* [] 表示重复的一个或多个内容
*
HisData File Structor
FileHead(84) + [HisDataRegion]
FileHead: dataTime(8)(FileTime)+dataTime(8)(LastUpdateTime)+DataRegionCount(4)+DatabaseName(64)
HisDataRegion Structor: RegionHead + DataBlockPoint Area + DataBlock Area
RegionHead: PreDataRegionPoint(8) + NextDataRegionPoint(8) + Datatime(8)+ tagcount(4)+ tagid sum(8)+file duration(4)+block duration(4)+Time tick duration(4)
DataBlockPoint Area: [ID]+[block Point]
[block point]: [[tag1 point,tag2 point,....][tag1 point,tag2 point,...].....] 以时间单位对变量的数去区指针进行组织
DataBlock Area: [block size + data block]
*/
///
///
///
public class DataFileInfo
{
#region ... Variables ...
private SortedDictionary> mTimeOffsets = new SortedDictionary>();
private bool mInited = false;
private static object mLockObj = new object();
private DateTime mLastTime;
private long mLastProcessOffset = -1;
///
///
///
private int mRegionCount = 0;
#endregion ...Variables...
#region ... Events ...
#endregion ...Events...
#region ... Constructor...
#endregion ...Constructor...
#region ... Properties ...
///
///
///
public DateTime LastTime
{
get
{
return mLastTime;
}
}
///
///
///
public string FId { get; set; }
///
///
///
public string FileName { get; set; }
///
/// 开始时间
///
public DateTime StartTime { get; set; }
///
///
///
public DateTime EndTime
{
get
{
return StartTime + Duration;
}
set
{
Duration = value - StartTime;
}
}
///
/// 时间长度
///
public TimeSpan Duration { get; set; }
#endregion ...Properties...
#region ... Methods ...
///
///
///
public void UpdateLastDatetime()
{
lock (mLockObj)
{
mTimeOffsets.Clear();
Scan();
if (DataFileManager.CurrentDateTime.ContainsKey(FId))
{
DataFileManager.CurrentDateTime[FId] = mLastTime;
}
else
{
DataFileManager.CurrentDateTime.Add(FId, mLastTime);
}
}
}
///
///
///
///
public void Scan()
{
long offset = DataFileManager.FileHeadSize;
DateTime time, tmp;
using (var ss = DataFileSeriserManager.manager.GetDefaultFileSersie())
{
ss.OpenForReadOnly(FileName);
//读取文件时间
DateTime fileTime = ss.ReadDateTime(0);
//读取最后一次更新时间
mLastTime = ss.ReadDateTime(8);
var rcount = ss.ReadInt(16);
if(rcount!= mRegionCount)
{
mRegionCount = rcount;
}
else
{
return;
}
do
{
//读取数据区时间
time = ss.ReadDateTime(offset + 16);
long oset = offset;
//读取下个区域位置
offset = ss.ReadLong(offset + 8);
if (offset != 0)
{
var dt2 = ss.ReadDateTime(offset + 16);
mTimeOffsets.Add(time, new Tuple(dt2 - time, oset, dt2));
tmp = dt2;
}
else
{
var tspan = StartTime + Duration - time;
if (tspan.TotalMilliseconds > 0)
mTimeOffsets.Add(time, new Tuple(tspan, oset, time + tspan));
tmp = time + tspan;
}
mLastProcessOffset = oset;
}
while (offset != 0);
if (mLastTime <= fileTime)
{
mLastTime = tmp;
}
//if (mLastProcessOffset == -1)
//{
//}
//else
//{
// offset = mLastProcessOffset;
// //读取数据区时间
// time = ss.ReadDateTime(offset + 16);
// long oset = offset;
// //读取下个区域位置
// offset = ss.ReadLong(offset + 8);
// if (offset != 0)
// {
// var dt2 = ss.ReadDateTime(offset + 16);
// if (mTimeOffsets.ContainsKey(time))
// {
// mTimeOffsets[time] = new Tuple(dt2 - time, oset, dt2);
// }
// else
// {
// mTimeOffsets.Add(time, new Tuple(dt2 - time, oset, dt2));
// }
// tmp = dt2;
// }
// else
// {
// var tspan = StartTime + Duration - time;
// if (tspan.TotalMilliseconds > 0)
// {
// if (mTimeOffsets.ContainsKey(time))
// {
// mTimeOffsets[time] = new Tuple(tspan, oset, time + tspan);
// }
// else
// {
// mTimeOffsets.Add(time, new Tuple(tspan, oset, time + tspan));
// }
// }
// tmp = time + tspan;
// }
//}
mInited = true;
}
}
///
///
///
///
///
public long GetFileOffsets(DateTime time)
{
lock (mLockObj)
if (!mInited) Scan();
foreach (var vv in mTimeOffsets)
{
if (vv.Key <= time && time < (vv.Key + vv.Value.Item1))
{
return vv.Value.Item2;
}
}
return -1;
}
///
///
///
///
///
///
public Dictionary> GetFileOffsets(DateTime startTime, DateTime endTime)
{
lock (mLockObj)
if (!mInited) Scan();
Dictionary> re = new Dictionary>();
foreach (var vv in mTimeOffsets)
{
if (vv.Key >= startTime && vv.Key < endTime)
{
re.Add(vv.Key, vv.Value);
}
}
return re;
}
///
/// 获取某个时间段内,不包括数据的时间段集合
///
///
///
///
public List> GetNoValueOffsets(DateTime startTime, DateTime endTime)
{
List> re = new List>();
DateTime stime = startTime;
List dtmp = new List();
DateTimeSpan ddt = new DateTimeSpan() { Start = startTime, End = endTime };
foreach (var vv in mTimeOffsets)
{
DateTimeSpan dts = new DateTimeSpan() { Start = vv.Key, End = vv.Value.Item3 };
dts = dts.Cross(ddt);
if (!dts.IsEmpty())
dtmp.Add(dts);
}
foreach (var vv in dtmp)
{
if (vv.Start > stime)
{
re.Add(new Tuple(stime, vv.Start));
}
stime = vv.End;
}
return re;
}
///
/// 判断某个时间点是否有数据
///
///
///
public bool HasValue(DateTime time)
{
foreach (var vv in mTimeOffsets)
{
if (vv.Key <= time && time < vv.Value.Item3)
{
return true;
}
}
return false;
}
#endregion ...Methods...
#region ... Interfaces ...
#endregion ...Interfaces...
}
///
///
///
public class DateTimeSpan
{
///
///
///
public DateTimeSpan Empty = new DateTimeSpan() { Start = DateTime.MinValue, End = DateTime.MinValue };
///
///
///
public DateTime Start { get; set; }
///
///
///
public DateTime End { get; set; }
///
///
///
///
public bool IsEmpty()
{
return Start == DateTime.MinValue && End == DateTime.MinValue;
}
///
///
///
///
public bool IsZore()
{
return (End - Start).TotalSeconds == 0;
}
///
///
///
///
///
public DateTimeSpan Cross(DateTimeSpan target)
{
DateTime stime = Max(target.Start, this.Start);
DateTime etime = Min(target.End, this.End);
if (etime < stime)
{
return Empty;
}
else
{
return new DateTimeSpan() { Start = stime, End = etime };
}
}
///
///
///
///
///
///
public DateTime Min(DateTime time1, DateTime time2)
{
return time1 <= time2 ? time1 : time2;
}
///
///
///
///
///
///
public DateTime Max(DateTime time1, DateTime time2)
{
return time1 >= time2 ? time1 : time2;
}
}
public static class DataFileInfoExtend
{
///
///
///
///
///
public static DataFileSeriserbase GetFileSeriser(this DataFileInfo file)
{
var re = DataFileSeriserManager.manager.GetDefaultFileSersie();
re.FileName = file.FileName;
re.OpenForReadOnly(file.FileName);
return re;
}
#region 读取所有值
///
/// 读取某时间段内的所有bool值
///
///
///
///
///
///
public static void ReadAllValue(this DataFileInfo file, int tid, DateTime startTime, DateTime endTime, HisQueryResult result)
{
var vff = file.GetFileSeriser();
//Dictionary> moffs = new Dictionary>();
var offset = file.GetFileOffsets(startTime, endTime);
foreach (var vv in offset)
{
DateTime stime = vv.Key > startTime ? vv.Key : startTime;
DateTime etime = vv.Key + vv.Value.Item1 > endTime ? endTime : vv.Key + vv.Value.Item1;
vff.ReadAllValue(vv.Value.Item2, tid, stime, etime, result);
}
vff.Close();
//GeneratorTime(moffs, startTime, endTime, file);
//foreach (var vf in moffs)
//{
// vff.ReadAllValue(vf.Key, tid, vf.Value.Item1, vf.Value.Item2, result);
//}
}
#endregion
#region 读取指定时刻值
///
///
///
///
///
///
///
///
///
public static object Read(this DataFileInfo file, int tid, DateTime time, QueryValueMatchType type)
{
using (var vff = file.GetFileSeriser())
{
var offset = file.GetFileOffsets(time);
return vff.Read(offset, tid, time, type);
}
}
///
///
///
///
///
///
///
///
///
public static HisQueryResult Read(this DataFileInfo file, int tid, List times, QueryValueMatchType type)
{
HisQueryResult re = new HisQueryResult(times.Count);
Read(file, tid, times, type, re);
return re;
}
///
///
///
///
///
///
///
///
///
public static void Read(this DataFileInfo file, int tid, List times, QueryValueMatchType type, HisQueryResult result)
{
using (var vff = file.GetFileSeriser())
{
Dictionary> moffs = new Dictionary>();
foreach (var vv in times)
{
var ff = file.GetFileOffsets(vv);
if (moffs.ContainsKey(ff))
{
moffs[ff].Add(vv);
}
else
{
moffs.Add(ff, new List() { vv });
}
}
foreach (var vf in moffs)
{
if (vf.Key > -1)
vff.Read(vf.Key, tid, vf.Value, type, result);
else
{
foreach (var vv in vf.Value)
{
result.Add(default(T), vv, (byte)QualityConst.Null);
}
}
}
}
}
#endregion
#region DataFileSeriser Read
///
///
///
///
///
///
///
///
///
///
public static void Read(this DataFileSeriserbase datafile, long offset, int tid, List dataTimes, QueryValueMatchType type, HisQueryResult res)
{
int timetick = 0;
var data = datafile.ReadTagDataBlock2(tid, offset, dataTimes, out timetick);
foreach (var vv in data)
{
DeCompressDataBlockValue(vv.Key, vv.Value, timetick, type, res);
}
foreach (var vv in data)
{
vv.Key.Dispose();
}
data.Clear();
}
///
///
///
///
///
///
///
///
///
///
public static object Read(this DataFileSeriserbase datafile, long offset, int tid, DateTime dataTime, QueryValueMatchType type)
{
int timetick = 0;
using (var data = datafile.ReadTagDataBlock(tid, offset, dataTime, out timetick))
{
return DeCompressDataBlockValue(data, dataTime, timetick, type);
}
}
///
///
///
///
///
///
///
///
///
public static void ReadAllValue(this DataFileSeriserbase datafile, long offset, int tid, DateTime startTime, DateTime endTime, HisQueryResult result)
{
int timetick = 0;
var data = datafile.ReadTagDataBlock2(tid, offset, startTime, endTime, out timetick);
foreach (var vv in data)
{
DeCompressDataBlockAllValue(vv.Key, vv.Value.Item1, vv.Value.Item2, timetick, result);
}
// System.Threading.Tasks.Parallel.ForEach(data, (vv) => { DeCompressDataBlockAllValue(vv.Key, vv.Value.Item1, vv.Value.Item2, timetick, result); });
foreach (var vv in data)
{
vv.Key.Dispose();
}
data.Clear();
}
#endregion
#region DeCompressData
///
///
///
///
///
///
///
///
///
private static object DeCompressDataBlockValue(MarshalMemoryBlock memory, DateTime datatime, int timeTick, QueryValueMatchType type)
{
//MarshalMemoryBlock target = new MarshalMemoryBlock(memory.Length);
//读取压缩类型
var ctype = memory.ReadByte();
var tp = CompressUnitManager2.Manager.GetCompress(ctype);
if (tp != null)
{
return tp.DeCompressValue(memory, 1, datatime, timeTick, type);
}
return null;
}
///
///
///
///
///
///
///
///
///
private static void DeCompressDataBlockValue(MarshalMemoryBlock memory, List datatime, int timeTick, QueryValueMatchType type, HisQueryResult result)
{
//MarshalMemoryBlock target = new MarshalMemoryBlock(memory.Length);
//读取压缩类型
var ctype = memory.ReadByte();
var tp = CompressUnitManager2.Manager.GetCompress(ctype);
if (tp != null)
{
tp.DeCompressValue(memory, 1, datatime, timeTick, type, result);
}
}
///
///
///
///
///
///
///
///
private static void DeCompressDataBlockAllValue(MarshalMemoryBlock memory, DateTime startTime, DateTime endTime, int timeTick, HisQueryResult result)
{
//MarshalMemoryBlock target = new MarshalMemoryBlock(memory.Length);
//读取压缩类型
var ctype = memory.ReadByte();
var tp = CompressUnitManager2.Manager.GetCompress(ctype);
if (tp != null)
{
tp.DeCompressAllValue(memory, 1, startTime, endTime, timeTick, result);
}
}
#endregion
#region 读取数据区域头数据
///
/// 检测数据头部指针区域数据是否被缓存
///
///
///
///
///
///
///
public static Dictionary CheckBlockHeadCach(this DataFileSeriserbase datafile, long offset, out int tagCount, out int fileDuration, out int blockDuration, out int timetick, out long blockPointer, out DateTime time)
{
//文件头部结构:Pre DataRegion(8) + Next DataRegion(8) + Datatime(8)+tagcount(4)+ tagid sum(8) +file duration(4)+ block duration(4)+Time tick duration(4)+ { + tagid1+tagid2+...+tagidn }+ {[tag1 block point1(8) + tag2 block point1+ tag3 block point1+...] + [tag1 block point2(8) + tag2 block point2+ tag3 block point2+...]....}
var dataoffset = offset + 16;
//读取时间
time = datafile.ReadDateTime(dataoffset);
dataoffset += 8;
//读取变量个数
int count = datafile.ReadInt(dataoffset);
dataoffset += 4;
tagCount = count;
//读取校验和
long idsum = datafile.ReadLong(dataoffset);
dataoffset += 8;
//读取单个文件的时长
fileDuration = datafile.ReadInt(dataoffset);
dataoffset += 4;
//读取数据块时长
blockDuration = datafile.ReadInt(dataoffset);
dataoffset += 4;
//读取时钟周期
timetick = datafile.ReadInt(dataoffset);
dataoffset += 4;
lock (TagHeadOffsetManager.manager)
{
if (!TagHeadOffsetManager.manager.Contains(idsum, count))
{
//Tag id 列表经过压缩,内容格式为:DataSize + Data
var dsize = datafile.ReadInt(dataoffset);
if (dsize <= 0)
{
tagCount = 0;
fileDuration = 0;
blockDuration = 0;
timetick = 0;
blockPointer = 0;
return new Dictionary();
}
dataoffset += 4;
blockPointer = dataoffset + dsize - offset;
var dtmp = new Dictionary();
using (var dd = datafile.Read(dataoffset, dsize))
{
MarshalVarintCodeMemory vcm = new MarshalVarintCodeMemory(dd.StartMemory, dsize);
var ltmp = vcm.ToIntList();
//vcm.Dispose();
if (ltmp.Count > 0)
{
int preid = ltmp[0];
dtmp.Add(preid, 0);
for (int i = 1; i < ltmp.Count; i++)
{
var id = ltmp[i] + preid;
dtmp.Add(id, i);
preid = id;
}
}
TagHeadOffsetManager.manager.Add(idsum, count, dtmp, blockPointer);
}
return dtmp;
}
else
{
var re = TagHeadOffsetManager.manager.Get(idsum, count);
blockPointer = re.Item2;
return re.Item1;
}
}
}
///
/// 读取某个变量在头部文件种的序号
///
///
///
///
///
///
///
///
public static int ReadTagIndexInDataPointer(this DataFileSeriserbase datafile, int tid, long offset, out int tagCount, out int fileDuration, out int blockDuration, out int timetick, out long blockpointer, out DateTime time)
{
var hfile = datafile.CheckBlockHeadCach(offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time);
if (hfile.ContainsKey(tid))
{
return hfile[tid];
}
return -1;
}
///
/// 读取数据区域的数据头数据
///
///
///
///
///
///
///
///
public static List ReadTargetBlockAddress(this DataFileSeriserbase datafile, List tid, long offset, out int tagCount, out int fileDuration, out int blockDuration, out int timetick, out long blockpointer, out DateTime time)
{
var hfile = datafile.CheckBlockHeadCach(offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time);
List re = new List();
foreach (var vv in tid)
{
if (hfile.ContainsKey(vv))
{
re.Add(hfile[vv]);
}
else
{
re.Add(-1);
}
}
return re;
}
#endregion
#region 读取数据块
///
///
///
///
///
///
///
///
///
public static MarshalMemoryBlock ReadTagDataBlock(this DataFileSeriserbase datafile, int tid, long offset, DateTime dataTime, out int timetick)
{
int fileDuration, blockDuration = 0;
int tagCount = 0;
DateTime time;
long blockpointer = 0;
var blockIndex = datafile.ReadTagIndexInDataPointer(tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time);
int blockcount = fileDuration * 60 / blockDuration;
var startTime = datafile.ReadDateTime(16);
var ttmp = (dataTime - startTime).TotalMinutes;
int dindex = (int)(ttmp / blockDuration);
if (ttmp % blockDuration > 0)
{
dindex++;
}
if (dindex > blockcount)
{
throw new Exception("DataPointer index is out of total block number");
}
var dataPointer = datafile.ReadLong(blockIndex * 8 + dindex * tagCount * 8); //读取DataBlock的地址
var datasize = datafile.ReadInt(dataPointer); //读取DataBlock 的大小
return datafile.Read(dataPointer + 4, datasize);
}
///
///
///
///
///
///
///
///
///
public static Dictionary ReadTagDataBlock(this DataFileSeriserbase datafile, int tid, long offset, List dataTimes, out int timetick)
{
int fileDuration, blockDuration = 0;
int tagCount = 0;
long blockpointer = 0;
DateTime time;
var blockIndex = datafile.ReadTagIndexInDataPointer(tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time);
int blockcount = fileDuration * 60 / blockDuration;
var startTime = datafile.ReadDateTime(16);
Dictionary rtmp = new Dictionary();
Dictionary re = new Dictionary();
foreach (var vdd in dataTimes)
{
var ttmp = (vdd - startTime).TotalMinutes;
int dindex = (int)(ttmp / blockDuration);
if (ttmp % blockDuration > 0)
{
dindex++;
}
if (dindex > blockcount)
{
throw new Exception("DataPointer index is out of total block number");
}
var dataPointer = datafile.ReadLong(blockIndex * 8 + dindex * tagCount * 8); //读取DataBlock的地址
var datasize = datafile.ReadInt(dataPointer); //读取DataBlock 的大小
if (!rtmp.ContainsKey(dataPointer))
{
var rmm = datafile.Read(dataPointer + 4, datasize);
re.Add(vdd, rmm);
rtmp.Add(dataPointer, rmm);
}
else
{
re.Add(vdd, rtmp[dataPointer]);
}
}
return re;
}
///
///
///
///
///
///
///
///
///
public static Dictionary> ReadTagDataBlock2(this DataFileSeriserbase datafile, int tid, long offset, List dataTimes, out int timetick)
{
int fileDuration, blockDuration = 0;
int tagCount = 0;
long blockpointer = 0;
DateTime time;
var tagIndex = datafile.ReadTagIndexInDataPointer(tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time);
Dictionary rtmp = new Dictionary();
Dictionary> re = new Dictionary>();
if (tagCount == 0) return re;
int blockcount = fileDuration * 60 / blockDuration;
var startTime = datafile.ReadDateTime(0);
//var startTime = time;
foreach (var vdd in dataTimes)
{
var ttmp = (vdd - startTime).TotalMinutes;
int blockindex = (int)(ttmp / blockDuration);
//if (ttmp % blockDuration > 0)
//{
// blockindex++;
//}
if (blockindex > blockcount)
{
throw new Exception("DataPointer index is out of total block number");
}
var dataPointer = datafile.ReadLong(offset + blockpointer + tagIndex * 8 + blockindex * tagCount * 8); //读取DataBlock的地址
if (dataPointer > 0)
{
var datasize = datafile.ReadInt(dataPointer); //读取DataBlock 的大小
if (datasize > 0)
{
//var rmm = datafile.Read(dataPointer + 4, (int)datasize);
//if (!re.ContainsKey(rmm))
//{
// re.Add(rmm, new Tuple(sstart, end));
//}
if (!rtmp.ContainsKey(dataPointer))
{
var rmm = datafile.Read(dataPointer + 4, datasize);
if (!re.ContainsKey(rmm))
{
re.Add(rmm, new List() { vdd });
}
else
{
re[rmm].Add(vdd);
}
rtmp.Add(dataPointer, rmm);
}
else
{
var rmm = rtmp[dataPointer];
if (!re.ContainsKey(rmm))
{
re.Add(rmm, new List() { vdd });
}
else
{
re[rmm].Add(vdd);
}
}
}
}
}
return re;
}
///
///
///
///
///
///
///
///
///
///
///
public static Dictionary> ReadTagDataBlock2(this DataFileSeriserbase datafile, int tid, long offset, DateTime start, DateTime end, out int timetick)
{
int fileDuration, blockDuration = 0;
int tagCount = 0;
long blockpointer = 0;
DateTime time;
var tagIndex = datafile.ReadTagIndexInDataPointer(tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time);
int blockcount = fileDuration * 60 / blockDuration;
//读取文件开始时间
var startTime = datafile.ReadDateTime(0);
//var startTime = time;
Dictionary> re = new Dictionary>();
DateTime sstart = start;
DateTime send = end;
while (sstart < end)
{
var ttmp = Math.Round((sstart - startTime).TotalSeconds, 3);
var vv = blockDuration * 60 - (ttmp % (blockDuration * 60));
send = sstart.AddSeconds(vv);
//send = (sstart - new TimeSpan(0, 0, 0, sstart.Second, sstart.Millisecond)).AddMinutes(blockDuration);
if (send > end)
{
send = end;
}
int blockindex = (int)(ttmp / (blockDuration * 60));
//if (ttmp % blockDuration > 0)
//{
// dindex++;
//}
if (blockindex > blockcount)
{
throw new Exception("DataPointer index is out of total block number");
}
var dataPointer = datafile.ReadLong(offset + blockpointer + tagIndex * 8 + blockindex * tagCount * 8); //读取DataBlock的地址
if (dataPointer > 0)
{
var datasize = datafile.ReadInt(dataPointer); //读取DataBlock 的大小
if (datasize > 0)
{
var rmm = datafile.Read(dataPointer + 4, (int)datasize);
if (!re.ContainsKey(rmm))
{
re.Add(rmm, new Tuple(sstart, send));
}
}
}
sstart = send;
}
return re;
}
#endregion
}
}