添加断点续传组件 和 修改热更下载组件 (#202)

* Create UnityWebRequestRenewalAsync.cs

添加断点续传组件
支持多任务同时下载，提高下载速度
Co-Authored-By: Nmaxiao <1173777+xiaohong@users.noreply.github.com>

* 添加断点续传组件

* Update UnityWebRequestRenewalAsync.cs

删除字节格式化为GMK格式方法

* 修改热更Bundle组件

添加多任务下载Bundle
            /*
                //最多同时下载n个文件 下载40M(400~500)个文件测试时间(ms)对比
                //等待n个任务同时完成再继续 1~61616 2~44796 3~34377 4~31918 5~27184 6~25564 7~22817 8~22719
                //完成1个补充1个最大n个任务 1~60309 8~11871 9~10843 10~10600 15~9309 20~9146 100~9195
                */

//最大任务数量20 速度从61秒提升到9秒
Co-Authored-By: Nmaxiao <1173777+xiaohong@users.noreply.github.com>

* 添加验证断点续传的文件服务器端是否有变化
Co-authored-by: Nmaxiao <1173777+xiaohong@users.noreply.github.com>

Unity/Assets/Model/Module/Resource/BundleDownloaderComponent.cs

@@ -12,7 +12,6 @@ namespace ET
 		{
 			self.bundles = new Queue<string>();
 			self.downloadedBundles = new HashSet<string>();
-			self.downloadingBundle = "";
 		}
 	}
 
@@ -29,9 +28,8 @@ namespace ET
 
 		public HashSet<string> downloadedBundles;
 
-		public string downloadingBundle;
-
-		public UnityWebRequestAsync webRequest;
+		//多任务同时下载
+		public List<UnityWebRequestAsync> webRequests = new List<UnityWebRequestAsync>();
 		
 		public override void Dispose()
 		{
@@ -51,8 +49,11 @@ namespace ET
 				this.TotalSize = 0;
 				this.bundles = null;
 				this.downloadedBundles = null;
-				this.downloadingBundle = null;
-				this.webRequest?.Dispose();
+				foreach (UnityWebRequestAsync webRequest in this.webRequests)
+				{
+					webRequest.Dispose();
+				}
+				webRequests.Clear();
 
 				this.Parent.RemoveComponent<BundleDownloaderComponent>();
 		}
@@ -136,14 +137,15 @@ namespace ET
 				}
 
 				long alreadyDownloadBytes = 0;
+				//已经下载完成的
 				foreach (string downloadedBundle in this.downloadedBundles)
 				{
-					long size = this.remoteVersionConfig.FileInfoDict[downloadedBundle].Size;
-					alreadyDownloadBytes += size;
+					alreadyDownloadBytes += this.remoteVersionConfig.FileInfoDict[downloadedBundle].Size;
 				}
-				if (this.webRequest != null)
+				//当前正在下载的
+				foreach (UnityWebRequestAsync webRequest in webRequests)
 				{
-					alreadyDownloadBytes += (long)this.webRequest.Request.downloadedBytes;
+					alreadyDownloadBytes += (long) webRequest.Request.downloadedBytes;
 				}
 				return (int)(alreadyDownloadBytes * 100f / this.TotalSize);
 			}
@@ -151,50 +153,69 @@ namespace ET
 
 		public async ETTask DownloadAsync(string url)
 		{
-			if (this.bundles.Count == 0 && this.downloadingBundle == "")
+			if (this.bundles.Count == 0)
 			{
 				return;
 			}
-
 			try
 			{
-				while (true)
-				{
-					if (this.bundles.Count == 0)
-					{
-						break;
-					}
-
-					this.downloadingBundle = this.bundles.Dequeue();
-
-					while (true)
-					{
-						try
-						{
-							using (this.webRequest = EntityFactory.Create<UnityWebRequestAsync>(this.Domain))
-							{
-								await this.webRequest.DownloadAsync(url + "StreamingAssets/" + this.downloadingBundle);
-								byte[] data = this.webRequest.Request.downloadHandler.data;
-
-								string path = Path.Combine(PathHelper.AppHotfixResPath, this.downloadingBundle);
-								using (FileStream fs = new FileStream(path, FileMode.Create))
-								{
-									fs.Write(data, 0, data.Length);
-								}
-							}
-						}
-						catch (Exception e)
-						{
-							Log.Error($"download bundle error: {this.downloadingBundle}\n{e}");
-							continue;
-						}
-
-						break;
-					}
-					this.downloadedBundles.Add(this.downloadingBundle);
-					this.downloadingBundle = "";
-					this.webRequest = null;
-				}
+				//正在下载的文件个数
+				int downloadingCount = 0;
+				//下载单个文件
+                async void downloadFile()
+                {
+                    if (this.bundles.Count == 0)
+                        return;
+                    downloadingCount++;
+                    //取出一个进行下载
+                    string downloading = this.bundles.Dequeue();
+                    Log.Debug($"开始下载({downloadingCount}):{downloading}");
+                    try
+                    {
+
+                        using (UnityWebRequestAsync webRequest = EntityFactory.Create<UnityWebRequestAsync>(this.Domain))
+                        {
+                            webRequests.Add(webRequest);
+                            await webRequest.DownloadAsync(url + "StreamingAssets/" + downloading);
+                            byte[] data = webRequest.Request.downloadHandler.data;
+                            webRequests.Remove(webRequest);
+                            string path = Path.Combine(PathHelper.AppHotfixResPath, downloading);
+                            using (FileStream fs = new FileStream(path, FileMode.Create))
+                            {
+	                            fs.Write(data, 0, data.Length);
+                            }
+                        }
+                    }
+                    catch (Exception e)
+                    {
+                        //下载异常跳过
+                        Log.Error($"download bundle error: {downloading}\n{e}");
+                    }
+                    finally
+                    {
+	                    downloadingCount--;
+                    }
+                    //正常下载
+                    this.downloadedBundles.Add(downloading);
+                    Log.Debug($"download bundle Finish: {downloading}\n");
+                }
+                /*
+                //最多同时下载n个文件 下载40M(400~500)个文件测试时间(ms)对比
+                //等待n个任务同时完成再继续 1~61616 2~44796 3~34377 4~31918 5~27184 6~25564 7~22817 8~22719
+                //完成1个补充1个最大n个任务 1~61309 8~11871 9~10843 10~10600 15~9309 20~9146 100~9195
+                */
+                
+                //最大任务数量20 速度从61秒提升到9秒
+                int maxCount = 20;
+                while (true)
+                {
+	                await TimerComponent.Instance.WaitAsync(10);
+	                //需要下载队列取完 正在下载为0表示完成更新
+	                if (this.bundles.Count == 0 && downloadingCount == 0)
+		                break;
+	                for (int i = downloadingCount; i < maxCount; i++)
+		                downloadFile();
+                }
 			}
 			catch (Exception e)
 			{

Unity/Assets/Model/Module/UnityWebRequest/UnityWebRequestRenewalAsync.cs

+using System;
+using System.Collections.Generic;
+using System.IO;
+using UnityEngine.Networking;
+
+namespace ET
+{
+    public class UnityWebRequestRenewalUpdateSystem: UpdateSystem<UnityWebRequestRenewalAsync>
+    {
+        public override void Update(UnityWebRequestRenewalAsync self)
+        {
+            self.Update();
+        }
+    }
+
+    /// <summary>
+    /// 断点续传实现
+    /// </summary>
+    public class UnityWebRequestRenewalAsync: Entity
+    {
+        public class AcceptAllCertificate: CertificateHandler
+        {
+            protected override bool ValidateCertificate(byte[] certificateData)
+            {
+                return true;
+            }
+        }
+
+        public static AcceptAllCertificate certificateHandler = new AcceptAllCertificate();
+
+        public UnityWebRequest headRequest;
+
+        public bool isCancel;
+
+        public ETTaskCompletionSource tcs;
+
+        //请求资源类型
+        private class RequestType
+        {
+            public const int None = 0; //未开始
+            public const int Head = 1; //文件头
+            public const int Data = 2; //文件数据
+        }
+
+        //当前请求资源类型
+        private int requestType = RequestType.None;
+
+        //多任务同时请求
+        private readonly List<UnityWebRequest> dataRequests = new List<UnityWebRequest>();
+
+        //当前下载的文件流 边下边存文件流
+        private FileStream fileStream;
+
+        //资源地址
+        private string Url;
+
+        //每个下载包长度
+        private int packageLength = 1000000; //1M
+
+        //同时开启任务最大个数
+        private int maxCount = 20;
+
+        //已经写入文件的大小
+        private long byteWrites;
+
+        //文件总大小
+        private long totalBytes;
+
+        //当前请求的位置
+        private long downloadIndex;
+
+        //文件下载是否出错
+        private string dataError
+        {
+            get
+            {
+                foreach (UnityWebRequest webRequest in this.dataRequests)
+                    if (!string.IsNullOrEmpty(webRequest.error))
+                        return webRequest.error;
+                return "";
+            }
+        }
+
+        //批量开启任务下载
+        void DownloadPackages()
+        {
+            if (dataRequests.Count >= maxCount || this.downloadIndex == totalBytes - 1)
+                return;
+
+            //开启一个下载任务
+            void DownloadPackage(long start, long end)
+            {
+                this.downloadIndex = end;
+                Log.Debug($"Request Data ({start}~{end}):{Url}");
+                UnityWebRequest request = UnityWebRequest.Get(Url);
+                dataRequests.Add(request);
+                request.certificateHandler = certificateHandler;
+                request.SetRequestHeader("Range", $"bytes={start}-{end}");
+                request.SendWebRequest();
+            }
+
+            //开启批量下载
+            for (int i = dataRequests.Count; i < maxCount; i++)
+            {
+                long start = this.byteWrites + i * packageLength;
+                long end   = this.byteWrites + (i + 1) * packageLength - 1;
+                if (end > this.totalBytes)
+                    end = this.totalBytes - 1;
+                DownloadPackage(start, end);
+                if (end == this.totalBytes - 1)
+                    break;
+            }
+        }
+
+        //一次批量下载完成后写文件
+        void WritePackages()
+        {
+            //写入单个包
+            void WritePackage(UnityWebRequest webRequest)
+            {
+                byte[] buff = webRequest.downloadHandler.data;
+                if (buff != null && buff.Length > 0)
+                {
+                    this.fileStream.Write(buff, 0, buff.Length);
+                    this.byteWrites += buff.Length;
+                }
+
+                Log.Debug($"write file Length:{byteWrites}");
+            }
+
+            //从第一个开始顺序写入
+            while (this.dataRequests.Count > 0 && dataRequests[0].isDone)
+            {
+                UnityWebRequest first = dataRequests[0];
+                dataRequests.RemoveAt(0);
+                WritePackage(first);
+                first.Dispose();
+            }
+        }
+
+        //更新文件体下载
+        void UpdatePackages()
+        {
+            if (this.isCancel)
+            {
+                this.tcs.SetException(new Exception($"request data error: {dataError}"));
+                return;
+            }
+
+            if (!string.IsNullOrEmpty(dataError))
+            {
+                this.tcs.SetException(new Exception($"request data error: {dataError}"));
+                return;
+            }
+
+            this.WritePackages();
+            if (this.byteWrites == this.totalBytes)
+                this.tcs.SetResult();
+            else
+                this.DownloadPackages();
+        }
+
+        //更新文件头下载
+        void UpdateHead()
+        {
+            if (this.isCancel)
+            {
+                this.tcs.SetException(new Exception($"request error: {this.headRequest.error}"));
+                return;
+            }
+
+            if (!this.headRequest.isDone)
+            {
+                return;
+            }
+
+            if (!string.IsNullOrEmpty(this.headRequest.error))
+            {
+                this.tcs.SetException(new Exception($"request error: {this.headRequest.error}"));
+                return;
+            }
+
+            this.tcs.SetResult();
+        }
+        
+        //检测是不是同一个文件
+        bool CheckSameFile(string modifiedTime)
+        {
+            string cacheValue = PlayerPrefs.GetString(Url);
+            string currentValue = this.totalBytes+"|"+modifiedTime;
+            if (cacheValue == currentValue)
+                return true;
+            PlayerPrefs.SetString(Url, currentValue);
+            PlayerPrefs.Save();
+            Log.Debug($"断点续传下载一个新的文件:{Url} cacheValue:{cacheValue} currentValue:{currentValue}");
+            return false;
+        }
+
+        /// <summary>
+        /// 断点续传入口
+        /// </summary>
+        /// <param name="url">文件下载地址</param>
+        /// <param name="filePath">文件写入路径</param>
+        /// <param name="packageLength">单个任务包体字节大小</param>
+        /// <param name="maxCount">同时开启最大任务个数</param>
+        /// <returns></returns>
+        public async ETTask DownloadAsync(string url, string filePath, int packageLength = 1000000, int maxCount = 20)
+        {
+            try
+            {
+                url                = url.Replace(" ", "%20");
+                this.Url           = url;
+                this.packageLength = packageLength;
+                this.maxCount      = maxCount;
+                Log.Debug("Web Request:" + url);
+
+                #region Download File Header
+
+                this.requestType = RequestType.Head;
+                //下载文件头
+                Log.Debug($"Request Head: {Url}");
+                this.tcs         = new ETTaskCompletionSource();
+                this.headRequest = UnityWebRequest.Head(Url);
+                this.headRequest.SendWebRequest();
+                await this.tcs.Task;
+                this.totalBytes = long.Parse(this.headRequest.GetResponseHeader("Content-Length"));
+                Log.Debug($"totalBytes: {this.totalBytes}");
+                this.headRequest?.Dispose();
+                this.headRequest = null;
+
+                #endregion
+
+                #region Check Local File
+
+                var dirPath = Path.GetDirectoryName(filePath);
+                //如果路径不存在就创建
+                dirPath.CreateDirectory();
+                //打开或创建
+                fileStream = new FileStream(filePath, FileMode.OpenOrCreate, FileAccess.Write);
+                //获取已下载长度
+                this.byteWrites = fileStream.Length;
+                //通过本地缓存的服务器文件修改时间和文件总长度检测服务器是否是同一个文件 不是同一个从头开始写入
+                if (!CheckSameFile(modifiedTime))
+                    this.byteWrites = 0;
+                Log.Debug($"byteWrites: {this.byteWrites}");
+                if (this.byteWrites == this.totalBytes)
+                {
+                    Log.Debug("已经下载完成2");
+                    return;
+                }
+
+                //设置开始写入位置
+                fileStream.Seek(this.byteWrites, SeekOrigin.Begin);
+
+                #endregion
+
+                #region Download File Data
+
+                //下载文件数据
+                requestType = RequestType.Data;
+                Log.Debug($"Request Data: {Url}");
+                this.tcs = new ETTaskCompletionSource();
+                this.DownloadPackages();
+                await this.tcs.Task;
+
+                #endregion
+            }
+            catch (Exception e)
+            {
+                Log.Error($"下载:{Url} Exception:{e}");
+                throw;
+            }
+        }
+
+        //下载进度
+        public float Progress
+        {
+            get
+            {
+                if (this.totalBytes == 0)
+                    return 0;
+                return (float) ((this.byteWrites + ByteDownloaded) / (double) this.totalBytes);
+            }
+        }
+
+        //当前任务已经下载的长度
+        public long ByteDownloaded
+        {
+            get
+            {
+                long length = 0;
+                foreach (UnityWebRequest dataRequest in this.dataRequests)
+                    length += dataRequest.downloadHandler.data.Length;
+                return length;
+            }
+        }
+
+        public void Update()
+        {
+            if (this.requestType == RequestType.Head)
+                this.UpdateHead();
+            if (this.requestType == RequestType.Data)
+                this.UpdatePackages();
+        }
+
+        public override void Dispose()
+        {
+            if (this.IsDisposed)
+            {
+                return;
+            }
+
+            base.Dispose();
+            headRequest?.Dispose();
+            headRequest = null;
+            foreach (UnityWebRequest dataRequest in this.dataRequests)
+                dataRequest.Dispose();
+            dataRequests.Clear();
+            this.fileStream?.Close();
+            this.fileStream?.Dispose();
+            this.fileStream = null;
+            this.isCancel   = false;
+        }
+    }
+}

Unity/Unity.Model.csproj

@@ -198,6 +198,7 @@
      <Compile Include="Assets\Model\Module\Resource\ResourcesComponent.cs" />
      <Compile Include="Assets\Model\Module\Resource\VersionConfig.cs" />
      <Compile Include="Assets\Model\Module\UnityWebRequest\UnityWebRequestAsync.cs" />
+     <Compile Include="Assets\Model\Module\UnityWebRequest\UnityWebRequestRenewalAsync.cs" />
      <Compile Include="Assets\Model\Session\SessionComponent.cs" />
      <Compile Include="Assets\Model\Unit\TurnComponent.cs" />
      <Compile Include="Assets\Model\Unit\Unit.cs" />