jay.chang/ibms-dome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[後端] 調整後台同步分批輸入資料(413 request content-length too large), 隱藏刪除所有auth_page程序, 修改api條件

Browse Source 
[前端] 調整後台同步分批輸入資料(413 request content-length too large)
This commit is contained in:
dev02 2024-01-15 15:29:23 +08:00
parent 9202b7edf8
commit 1144d1a47e
7 changed files with 274 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Backend/Controllers/BuildMenuController.cs
View File

@ -254,19 +254,19 @@ namespace Backend.Controllers
await backendRepository.AddOneByCustomTable(pagedictionary, "auth_page");
var param = new { main_system_type = main_system_type, sub_system_type = sub_system_type };
await backendRepository.ExecuteSql(@"DELETE FROM auth_page
WHERE auth_page.AuthCode like 'F%';
INSERT INTO auth_page (AuthCode,AuthType,MainName,SubName,building_tag,ShowView)
SELECT concat('F', CONVERT(ROW_NUMBER() OVER(ORDER BY bm.building_tag ASC), nchar)) AuthCode, '1' as AuthType,
mv.system_key MainName, sv.system_key SubName, bm.building_tag, sv.id ShowView
FROM building_menu bm
LEFT JOIN variable mv ON bm.main_system_tag = mv.system_value AND mv.system_type = @main_system_type
LEFT JOIN variable sv ON bm.sub_system_tag = sv.system_value AND sv.system_type = @sub_system_type", param);
//await backendRepository.ExecuteSql(@"DELETE FROM auth_page
// WHERE auth_page.AuthCode like 'F%';
// INSERT INTO auth_page (AuthCode,AuthType,MainName,SubName,building_tag,ShowView)
// SELECT concat('F', CONVERT(ROW_NUMBER() OVER(ORDER BY bm.building_tag ASC), nchar)) AuthCode, '1' as AuthType,
// mv.system_key MainName, sv.system_key SubName, bm.building_tag, sv.id ShowView
// FROM building_menu bm
// LEFT JOIN variable mv ON bm.main_system_tag = mv.system_value AND mv.system_type = @main_system_type
// LEFT JOIN variable sv ON bm.sub_system_tag = sv.system_value AND sv.system_type = @sub_system_type", param);
await backendRepository.ExecuteSql(@"delete a from role_auth a join role b on a.role_guid = b.role_guid where b.layer = 0;
INSERT INTO role_auth (role_guid,AuthCode,created_by)
SELECT r.role_guid,ap.AuthCode,'0' created_by FROM auth_page ap,role r
WHERE r.layer = 0;");
//await backendRepository.ExecuteSql(@"delete a from role_auth a join role b on a.role_guid = b.role_guid where b.layer = 0;
// INSERT INTO role_auth (role_guid,AuthCode,created_by)
// SELECT r.role_guid,ap.AuthCode,'0' created_by FROM auth_page ap,role r
// WHERE r.layer = 0;");
apiResult.Code = "0000";
apiResult.Msg = "儲存成功";

160
Backend/Controllers/NiagaraDataSynchronizeController.cs
View File

@ -28,6 +28,7 @@ namespace Backend.Controllers
private Stopwatch stopwatchSection = new Stopwatch();
private Stopwatch syncStopWatch = new Stopwatch();
private static long syncWatchTime = 0;
public static bool isFirstAsync = false;
public NiagaraDataSynchronizeController(IBackendRepository backendRepository, INiagaraDataSynchronizeRepository niagaraDataSynchronizeRepository,IBackEndConfigHelper backEndConfig)
{
@ -280,63 +281,20 @@ namespace Backend.Controllers
/// </summary>
/// <returns></returns>
[HttpPost]
public async Task<ApiResult<bool>> DevIteComData([FromBody] List<ImpNiaItem> ds)
public async Task<ApiResult<bool>> DevIteComData()
{
syncStopWatch = new Stopwatch();
syncStopWatch.Start();
stopwatch = new Stopwatch();
stopwatch.Start();
bool result = false;
var oldDs = JsonConvert.DeserializeObject<List<ImpNiaItem>>(JsonConvert.SerializeObject(ds));
//var oldDs = JsonConvert.DeserializeObject<List<ImpNiaItem>>(JsonConvert.SerializeObject(ds));
ApiResult<bool> apiResult = new ApiResult<bool>();
try
{
if (ds.Count > 0)
{
//string ss = ds.Where(x => x.tag_name != "").FirstOrDefault().tag_name;
//var building = ss.Split("_")[0];
var building = ds.GroupBy(x => x.device_building_tag).Select(x => x.Key).ToList();
#region 歷史資料處理
ds.Clear();
var isDome = await backendRepository.GetOneAsync<string>("select system_value from variable where deleted = 0 and system_type = 'project_name';");
var sqlObix = $@"SELECT system_value as Value, system_key as Name FROM variable WHERE deleted = 0 AND system_type = 'obixConfig'";
var variableObix = backendRepository.GetAllAsync<Backend.Models.KeyValue>(sqlObix).Result;
var obixApiConfig = new Backend.Models.ObixApiConfig();
EDFunction ed = new EDFunction();
obixApiConfig.ApiBase = variableObix.Where(x => x.Name == "ApiBase").Select(x => x.Value).FirstOrDefault();
obixApiConfig.UserName = variableObix.Where(x => x.Name == "UserName").Select(x => x.Value).FirstOrDefault();
obixApiConfig.Password = variableObix.Where(x => x.Name == "Password").Select(x => x.Value).FirstOrDefault();
//obixApiConfig.UrlSlot = variableObix.Where(x => x.Name == "url_slot").Select(x => x.Value).FirstOrDefault();
webRequestService svc = new webRequestService();
stopwatchSection = new Stopwatch();
stopwatchSection.Start();
var data = svc.obixHisQuery_v2(obixApiConfig.ApiBase + "obix/config/Program/ObixQuery/query/", obixApiConfig.ApiBase + "obix/histories", "",
obixApiConfig.UserName, obixApiConfig.Password);
ds.AddRange(data);
stopwatchSection.Stop();
await niagaraDataSynchronizeRepository.KeepTimeLog("Get history device point from niagara", stopwatchSection.ElapsedMilliseconds);
// save to history
stopwatchSection = new Stopwatch();
stopwatchSection.Start();
await niagaraDataSynchronizeRepository.InsertItemFromNiagara_History(ds, building); // insert 暫存table import_niagara_item_history
stopwatchSection.Stop();
await niagaraDataSynchronizeRepository.KeepTimeLog("InsertItemFromNiagara_History", stopwatchSection.ElapsedMilliseconds);
if (!string.IsNullOrEmpty(isDome) && isDome.Contains("ibms_dome"))
{
stopwatchSection = new Stopwatch();
stopwatchSection.Start();
await niagaraDataSynchronizeRepository.RecoverSysTag("import_niagara_item_history"); // recover missing deivce_system_tag
stopwatchSection.Stop();
await niagaraDataSynchronizeRepository.KeepTimeLog("RecoverSysTag import_niagara_item_history", stopwatchSection.ElapsedMilliseconds);
}
#endregion
stopwatchSection = new Stopwatch();
stopwatchSection.Start();
await niagaraDataSynchronizeRepository.InsertItemFromNiagara(oldDs, building); // insert 暫存table import_niagara_item
stopwatchSection.Stop();
await niagaraDataSynchronizeRepository.KeepTimeLog("InsertItemFromNiagara", stopwatchSection.ElapsedMilliseconds);
//var building = ds.GroupBy(x => x.device_building_tag).Select(x => x.Key).ToList();
stopwatchSection = new Stopwatch();
stopwatchSection.Start();
@ -362,15 +320,16 @@ namespace Backend.Controllers
stopwatchSection.Stop();
await niagaraDataSynchronizeRepository.KeepTimeLog("CheckItemIsShowHistory", stopwatchSection.ElapsedMilliseconds);
result = true;
if (isFirstAsync)
isFirstAsync = false;
result = true;
apiResult.Code = "0000";
apiResult.Data = result;
}
}
catch (Exception exception)
{
isFirstAsync = false;
apiResult.Code = "9999";
apiResult.Msg = "【" + controllerName + "/" + actionName + "】" + exception.Message;
Logger.LogError("【" + controllerName + "/" + actionName + "】" + exception.Message + Environment.NewLine + exception.StackTrace);
@ -415,5 +374,108 @@ namespace Backend.Controllers
stopwatchSection.Stop();
await niagaraDataSynchronizeRepository.KeepTimeLog("DeviceDisasterAsync", stopwatchSection.ElapsedMilliseconds);
}
/// <summary>
/// 歷史資料處理
/// </summary>
/// <returns></returns>
[HttpPost]
public async Task<ApiResult<bool>> GetHistoryData()
{
ApiResult<bool> apiResult = new ApiResult<bool>();
apiResult.Data = false;
apiResult.Code = "9999";
try
{
#region 歷史資料處理
List<ImpNiaItem> ds = new List<ImpNiaItem>();
var building = await backendRepository.GetAllAsync<string>("select building_tag from building where deleted = 0");
var isDome = await backendRepository.GetOneAsync<string>("select system_value from variable where deleted = 0 and system_type = 'project_name';");
var sqlObix = $@"SELECT system_value as Value, system_key as Name FROM variable WHERE deleted = 0 AND system_type = 'obixConfig'";
var variableObix = backendRepository.GetAllAsync<Backend.Models.KeyValue>(sqlObix).Result;
var obixApiConfig = new Backend.Models.ObixApiConfig();
EDFunction ed = new EDFunction();
obixApiConfig.ApiBase = variableObix.Where(x => x.Name == "ApiBase").Select(x => x.Value).FirstOrDefault();
obixApiConfig.UserName = variableObix.Where(x => x.Name == "UserName").Select(x => x.Value).FirstOrDefault();
obixApiConfig.Password = variableObix.Where(x => x.Name == "Password").Select(x => x.Value).FirstOrDefault();
//obixApiConfig.UrlSlot = variableObix.Where(x => x.Name == "url_slot").Select(x => x.Value).FirstOrDefault();
webRequestService svc = new webRequestService();
stopwatchSection = new Stopwatch();
stopwatchSection.Start();
var data = svc.obixHisQuery_v2(obixApiConfig.ApiBase + "obix/config/Program/ObixQuery/query/", obixApiConfig.ApiBase + "obix/histories", "",
obixApiConfig.UserName, obixApiConfig.Password);
ds.AddRange(data);
stopwatchSection.Stop();
await niagaraDataSynchronizeRepository.KeepTimeLog("Get history device point from niagara", stopwatchSection.ElapsedMilliseconds);
// save to history
stopwatchSection = new Stopwatch();
stopwatchSection.Start();
await niagaraDataSynchronizeRepository.InsertItemFromNiagara_History(ds, building); // insert 暫存table import_niagara_item_history
stopwatchSection.Stop();
await niagaraDataSynchronizeRepository.KeepTimeLog("InsertItemFromNiagara_History", stopwatchSection.ElapsedMilliseconds);
if (!string.IsNullOrEmpty(isDome) && isDome.Contains("ibms_dome"))
{
stopwatchSection = new Stopwatch();
stopwatchSection.Start();
await niagaraDataSynchronizeRepository.RecoverSysTag("import_niagara_item_history"); // recover missing deivce_system_tag
stopwatchSection.Stop();
await niagaraDataSynchronizeRepository.KeepTimeLog("RecoverSysTag import_niagara_item_history", stopwatchSection.ElapsedMilliseconds);
}
#endregion
apiResult.Code = "0000";
apiResult.Data = true;
isFirstAsync = true;
}
catch (Exception exception)
{
apiResult.Code = "9999";
apiResult.Msg = "【" + controllerName + "/" + actionName + "】" + exception.Message;
Logger.LogError("【" + controllerName + "/" + actionName + "】" + exception.Message + Environment.NewLine + exception.StackTrace);
}
return apiResult;
}
/// <summary>
/// 設備點位處理
/// </summary>
/// <returns></returns>
[HttpPost]
public async Task<ApiResult<bool>> DevItemInsert([FromBody] List<ImpNiaItem> ds)
{
var oldDs = JsonConvert.DeserializeObject<List<ImpNiaItem>>(JsonConvert.SerializeObject(ds));
ApiResult<bool> apiResult = new ApiResult<bool>();
apiResult.Data = false;
apiResult.Code = "9999";
try
{
if (ds.Count > 0)
{
var building = await backendRepository.GetAllAsync<string>("select building_tag from building where deleted = 0");
stopwatchSection = new Stopwatch();
stopwatchSection.Start();
await niagaraDataSynchronizeRepository.InsertItemFromNiagara(oldDs, building, isFirstAsync); // insert 暫存table import_niagara_item
stopwatchSection.Stop();
await niagaraDataSynchronizeRepository.KeepTimeLog("InsertItemFromNiagara", stopwatchSection.ElapsedMilliseconds);
if (isFirstAsync)
isFirstAsync = false;
}
apiResult.Code = "0000";
apiResult.Data = true;
}
catch (Exception exception)
{
apiResult.Code = "9999";
apiResult.Msg = "【" + controllerName + "/" + actionName + "】" + exception.Message;
Logger.LogError("【" + controllerName + "/" + actionName + "】" + exception.Message + Environment.NewLine + exception.StackTrace);
}
return apiResult;
}
}
}

6
Backend/Startup.cs
View File

@ -22,6 +22,7 @@ using Microsoft.IdentityModel.Tokens;
using System.Text;
using System.IdentityModel.Tokens.Jwt;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Http.Features;
namespace Backend
{
@ -155,6 +156,11 @@ namespace Backend
{
options.IdleTimeout = TimeSpan.FromMinutes(loginExpireMinute);
});
services.Configure<FormOptions>(options =>
{
options.MultipartBodyLengthLimit = 1_073_741_824; //accept request content_length at 1gb
});
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.

79
Backend/Views/NiagaraDataSynchronize/Index.cshtml
View File

@ -185,9 +185,16 @@
{
//比對資料，有差異的話，再同步到device等資料表
var url_synchronize_data = "/NiagaraDataSynchronize/RawDataDevItemList/";
var url_synchronize_data_device_item = "/NiagaraDataSynchronize/DevIteComData/";
var url_synchronize_data_device_item = "/NiagaraDataSynchronize/DevItemInsert/";
let object = {};
let dataLength = 0;
let curData = {};
let asyncSendData = [];
let contentLength = 5000;
object.building = building;
//device async
$.ajax({
method: "POST",
url: url_synchronize_data,
@ -204,10 +211,36 @@
return;
}
//history async
$.ajax({
method: "POST",
url: '/NiagaraDataSynchronize/GetHistoryData/',
cache: false,
async: false,
contentType: "application/json; charset=UTF-8",
success: function (res) {
if (res.code != "0000") {
hideSpinner();
toast_error(res.msg);
document.getElementById('loadDataText').innerText = "比對資料出錯了!";
return;
}
curData = rel.data;
let relDataLength = curData.length;
$.each(rel.data, function (index, val) {
if (relDataLength > 0) {
//push data, counting
asyncSendData.push(val);
dataLength++;
relDataLength--;
if (dataLength >= contentLength || relDataLength <= 0) {
//device_item async-insert
$.ajax({
method: "POST",
url: url_synchronize_data_device_item,
data: JSON.stringify(rel.data),
data: JSON.stringify(asyncSendData),
cache: false,
async: false,
contentType: "application/json; charset=UTF-8",
@ -220,6 +253,26 @@
return;
}
//reset variable
dataLength = 0;
asyncSendData = [];
//device_item async-compare
if (relDataLength <= 0) {
$.ajax({
method: "POST",
url: '/NiagaraDataSynchronize/DevIteComData/',
cache: false,
async: false,
contentType: "application/json; charset=UTF-8",
success: function (rel) {
if (rel.code != "0000") {
toast_error(rel.msg);
hideSpinner();
document.getElementById('loadDataText').innerText = "比對資料出錯了!";
return;
}
hideSpinner();
console.log(rel);
var time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
@ -228,8 +281,30 @@
},
error: function () {
hideSpinner();
document.getElementById('loadDataText').innerText = "比對資料出錯了!";
return;
}
})
}
},
error: function () {
hideSpinner();
document.getElementById('loadDataText').innerText = "比對資料出錯了!";
return;
}
})
}
}
});
},
error: function () {
hideSpinner();
document.getElementById('loadDataText').innerText = "比對資料出錯了!";
return;
}
})
var time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
var dateTime = date + ' ' + time;
document.getElementById('loadDataText').innerText = "比對完成 \n" + dateTime;

2
FrontendWebApi/ApiControllers/HistoryController.cs
View File

@ -570,7 +570,7 @@ namespace FrontendWebApi.ApiControllers
join variable v1 on v1.system_value = b.device_system_tag and v1.deleted = 0 and v1.system_type = 'device_system_category_layer2'
join variable v2 on v2.system_value = b.device_name_tag and v2.deleted = 0 and v2.system_type = 'device_system_category_layer3'
where a.deleted = 0 and b.deleted = 0 and v2.deleted = 0 and v1.deleted = 0
and a.unit is not null and b.device_number in @Device_number and a.is_link = 1
and a.unit is not null and b.device_number in @Device_number and a.is_link = 1 and a.is_show_history = 1
order by c.priority,b.priority;"
, new { Device_number = postDevice.select_data.Select(a => a.device_number).ToList() }
);

5
Repository/BackendRepository/Implement/NiagaraDataSynchronizeRepository.cs
View File

@ -235,7 +235,7 @@ namespace Repository.BackendRepository.Implement
/// <param name="ds"></param>
/// <param name="building"></param>
/// <returns></returns>
public async Task InsertItemFromNiagara(List<ImpNiaItem> ds, List<string> building)
public async Task InsertItemFromNiagara(List<ImpNiaItem> ds, List<string> building, bool isFirstAsync)
{
stopwatch = new Stopwatch();
stopwatch.Start();
@ -245,6 +245,8 @@ namespace Repository.BackendRepository.Implement
try
{
#region 刪除 import_niagara_item資料表中選取的棟別
if (isFirstAsync)
{
stopwatchSection = new Stopwatch();
stopwatchSection.Start();
foreach(var b in building)
@ -273,6 +275,7 @@ namespace Repository.BackendRepository.Implement
}
stopwatchSection.Stop();
await KeepTimeLog("Create or delete related building import_niagara_tag", stopwatchSection.ElapsedMilliseconds);
}
#endregion
stopwatchSection = new Stopwatch();

2
Repository/BackendRepository/Interface/INiagaraDataSynchronizeRepository.cs
View File

@ -29,7 +29,7 @@ namespace Repository.BackendRepository.Interface
/// <param name="ds"></param>
/// <param name="building"></param>
/// <returns></returns>
Task InsertItemFromNiagara(List<ImpNiaItem> ds, List<string> building);
Task InsertItemFromNiagara(List<ImpNiaItem> ds, List<string> building, bool isFirstAsync);
/// <summary>
/// 比對 device
/// </summary>