Neuron®
The Neuron® is the basis for the creation of open and secure federated networks for smart societies.
Loading...
Searching...
No Matches
RestoreConfiguration.cs
1using System;
2using System.Collections.Generic;
3using System.IO;
4using System.IO.Compression;
5using System.Security.Cryptography;
6using System.Text;
7using System.Threading.Tasks;
8using System.Xml;
9using System.Xml.Xsl;
10using Waher.Content;
13using Waher.Events;
22using Waher.Script;
23
25{
29 public class RestoreConfiguration : SystemConfiguration, IDisposable
30 {
31 private static RestoreConfiguration instance = null;
32
33 private HttpResource uploadBackup = null;
34 private HttpResource uploadKey = null;
35 private HttpResource restore = null;
36
37 private readonly Dictionary<string, TemporaryFile> backupFilePerSession = new Dictionary<string, TemporaryFile>();
38 private readonly Dictionary<string, TemporaryFile> keyFilePerSession = new Dictionary<string, TemporaryFile>();
39 private int expectedBlockBackup = 0;
40 private int expectedBlockKey = 0;
41 private bool reloadConfiguration = false;
42
46 public static RestoreConfiguration Instance => instance;
47
51 public override string Resource => "/Settings/Restore.md";
52
56 public override int Priority => 150;
57
63 public override Task<string> Title(Language Language)
64 {
65 return Language.GetStringAsync(typeof(Gateway), 9, "Restore");
66 }
67
71 public override Task ConfigureSystem()
72 {
73 return Task.CompletedTask;
74 }
75
80 public override void SetStaticInstance(ISystemConfiguration Configuration)
81 {
82 instance = Configuration as RestoreConfiguration;
83 }
84
89 public override Task InitSetup(HttpServer WebServer)
90 {
91 this.uploadBackup = WebServer.Register("/Settings/UploadBackup", null, this.UploadBackup, true, false, true);
92 this.uploadKey = WebServer.Register("/Settings/UploadKey", null, this.UploadKey, true, false, true);
93 this.restore = WebServer.Register("/Settings/Restore", null, this.Restore, true, false, true);
94
95 WebServer.SessionRemoved += this.WebServer_SessionRemoved;
96
97 return base.InitSetup(WebServer);
98 }
99
104 public override Task UnregisterSetup(HttpServer WebServer)
105 {
106 WebServer.Unregister(this.uploadBackup);
107 WebServer.Unregister(this.uploadKey);
108 WebServer.Unregister(this.restore);
109
110 WebServer.SessionRemoved -= this.WebServer_SessionRemoved;
111
112 return base.UnregisterSetup(WebServer);
113 }
114
115 private Task WebServer_SessionRemoved(object Sender, CacheItemEventArgs<string, Variables> e)
116 {
117 RemoveFile(e.Key, this.backupFilePerSession);
118 RemoveFile(e.Key, this.keyFilePerSession);
119
120 return Task.CompletedTask;
121 }
122
123 private static void RemoveFile(string Key, Dictionary<string, TemporaryFile> Files)
124 {
125 lock (Files)
126 {
127 if (Files.TryGetValue(Key, out TemporaryFile File))
128 {
129 Files.Remove(Key);
130 File.Dispose();
131 }
132 }
133 }
134
135 private Task UploadBackup(HttpRequest Request, HttpResponse Response)
136 {
137 this.Upload(Request, Response, ref this.expectedBlockBackup, this.backupFilePerSession, "backup");
138 return Task.CompletedTask;
139 }
140
141 private Task UploadKey(HttpRequest Request, HttpResponse Response)
142 {
143 this.Upload(Request, Response, ref this.expectedBlockKey, this.keyFilePerSession, "key");
144 return Task.CompletedTask;
145 }
146
150 protected override string ConfigPrivilege => "Admin.Data.Restore";
151
152 private void Upload(HttpRequest Request, HttpResponse Response, ref int ExpectedBlockNr, Dictionary<string, TemporaryFile> Files, string Name)
153 {
154 Gateway.AssertUserAuthenticated(Request, this.ConfigPrivilege);
155
156 TemporaryFile File;
157 string TabID;
158 string HttpSessionID;
159
160 if (!Request.HasData ||
161 !Request.Header.TryGetHeaderField("X-TabID", out HttpField F) ||
162 string.IsNullOrEmpty(TabID = F.Value) ||
163 !Request.Header.TryGetHeaderField("X-BlockNr", out F) ||
164 !int.TryParse(F.Value, out int BlockNr) ||
165 !Request.Header.TryGetHeaderField("X-More", out F) ||
166 !CommonTypes.TryParse(F.Value, out bool More) ||
167 string.IsNullOrEmpty(HttpSessionID = HttpResource.GetSessionId(Request, Response)))
168 {
169 throw new BadRequestException();
170 }
171
172 if (BlockNr == 0)
173 {
174 ExpectedBlockNr = 0;
175 RemoveFile(HttpSessionID, Files);
176
177 if (Request.Header.TryGetHeaderField("X-FileName", out F) && !string.IsNullOrEmpty(F.Value))
178 Request.Session[Name + "FileName"] = F.Value;
179 else
180 throw new BadRequestException();
181 }
182
183 if (BlockNr != ExpectedBlockNr)
184 throw new BadRequestException();
185
186 ExpectedBlockNr++;
187
188 lock (Files)
189 {
190 if (!Files.TryGetValue(HttpSessionID, out File))
191 {
192 File = new TemporaryFile();
193 Files[HttpSessionID] = File;
194 }
195 }
196
197 Request.DataStream.CopyTo(File);
198
199 if (!More)
200 File.Flush();
201
202 ShowStatus(TabID, Name + "Bytes", Export.FormatBytes(File.Length) + " received of " + Name + " file.");
203
204 Response.StatusCode = 200;
205 }
206
207 internal static string[] GetTabIDs(string TabID)
208 {
209 if (string.IsNullOrEmpty(TabID))
210 return ClientEvents.GetTabIDs();
211 else
212 return new string[] { TabID };
213 }
214
215 private static void CollectionFound(string TabID, string CollectionName)
216 {
217 ClientEvents.PushEvent(GetTabIDs(TabID), "CollectionFound", CollectionName, false);
218 }
219
220 private static void ShowStatus(string TabID, string Id, string Message)
221 {
222 ClientEvents.PushEvent(GetTabIDs(TabID), "ShowStatus", JSON.Encode(new Dictionary<string, object>()
223 {
224 { "id", Id },
225 { "message", Message },
226 }, false), true);
227 }
228
229 private static void ShowStatus(string TabID, string Message)
230 {
231 ClientEvents.PushEvent(GetTabIDs(TabID), "ShowStatus", Message, false);
232 }
233
234 private async Task Restore(HttpRequest Request, HttpResponse Response)
235 {
236 Gateway.AssertUserAuthenticated(Request, this.ConfigPrivilege);
237
238 TemporaryFile BackupFile;
239 TemporaryFile KeyFile;
240 string TabID;
241 string HttpSessionID;
242
243 if (!Request.HasData ||
244 !Request.Header.TryGetHeaderField("X-TabID", out HttpField F) ||
245 string.IsNullOrEmpty(TabID = F.Value) ||
246 string.IsNullOrEmpty(HttpSessionID = HttpResource.GetSessionId(Request, Response)))
247 {
248 throw new BadRequestException();
249 }
250
251 object Obj = await Request.DecodeDataAsync();
252 if (!(Obj is Dictionary<string, object> Parameters))
253 throw new BadRequestException();
254
255 if (!Parameters.TryGetValue("overwrite", out Obj) || !(Obj is bool Overwrite) ||
256 !Parameters.TryGetValue("onlySelectedCollections", out Obj) || !(Obj is bool OnlySelectedCollections) ||
257 !Parameters.TryGetValue("selectedCollections", out Obj) || !(Obj is Array SelectedCollections) ||
258 !Parameters.TryGetValue("selectedParts", out Obj) || !(Obj is Array SelectedParts))
259 {
260 throw new BadRequestException();
261 }
262
263 BackupFile = GetAndRemoveFile(HttpSessionID, this.backupFilePerSession);
264 KeyFile = GetAndRemoveFile(HttpSessionID, this.keyFilePerSession);
265
266 Task _ = Task.Run(async () => await this.Restore(BackupFile, KeyFile, TabID, Request.Session["backupFileName"]?.ToString(),
267 Overwrite, OnlySelectedCollections, SelectedCollections, SelectedParts));
268
269 Response.StatusCode = 200;
270 }
271
272 private static TemporaryFile GetAndRemoveFile(string SessionID, Dictionary<string, TemporaryFile> Files)
273 {
274 lock (Files)
275 {
276 if (Files.TryGetValue(SessionID, out TemporaryFile File))
277 {
278 Files.Remove(SessionID);
279 return File;
280 }
281 else
282 return null;
283 }
284 }
285
289 public void Dispose()
290 {
291 Clear(this.backupFilePerSession);
292 Clear(this.keyFilePerSession);
293 }
294
295 private static void Clear(Dictionary<string, TemporaryFile> Files)
296 {
297 if (!(Files is null))
298 {
299 lock (Files)
300 {
301 foreach (TemporaryFile File in Files.Values)
302 File.Dispose();
303
304 Files.Clear();
305 }
306 }
307 }
308
309 private async Task Restore(FileStream BackupFile, FileStream KeyFile, string TabID, string BackupFileName, bool Overwrite,
310 bool OnlySelectedCollections, Array SelectedCollections, Array SelectedParts)
311 {
312 ICryptoTransform AesTransform1 = null;
313 ICryptoTransform AesTransform2 = null;
314 CryptoStream cs1 = null;
315 CryptoStream cs2 = null;
316
317 try
318 {
319 if (Overwrite)
320 ShowStatus(TabID, "Restoring backup.");
321 else
322 ShowStatus(TabID, "Validating backup.");
323
324 if (BackupFile is null || string.IsNullOrEmpty(BackupFileName))
325 throw new Exception("No backup file selected.");
326
327 string Extension = Path.GetExtension(BackupFileName);
328 ValidateBackupFile Import = new ValidateBackupFile(BackupFileName, null);
329
330 (AesTransform1, cs1) = await DoImport(BackupFile, KeyFile, TabID, Extension, Import, false,
331 false, new string[0], new string[0]);
332
333 if (Overwrite)
334 {
335 ShowStatus(TabID, "Restoring backup.");
336 Import = new RestoreBackupFile(BackupFileName, Import.ObjectIdMap);
337
338 (AesTransform2, cs2) = await DoImport(BackupFile, KeyFile, TabID, Extension, Import, true,
339 OnlySelectedCollections, SelectedCollections, SelectedParts);
340
341 this.reloadConfiguration = true;
342 await DoAnalyze(TabID);
343
344 Caches.ClearAll(false);
345 }
346
347 StringBuilder Result = new StringBuilder();
348
349 if (Overwrite)
350 Result.AppendLine("Restoration complete.");
351 else
352 Result.AppendLine("Verification complete.");
353
354 if (Import.NrCollections > 0 || Import.NrObjects > 0 || Import.NrProperties > 0 || Import.NrFiles > 0)
355 {
356 Result.AppendLine();
357 Result.AppendLine("Contents of file:");
358 Result.AppendLine();
359
360 if (Import.NrCollections > 0)
361 {
362 Result.Append(Import.NrCollections.ToString());
363 if (Import.NrCollections > 1)
364 Result.AppendLine(" collections.");
365 else
366 Result.AppendLine(" collection.");
367 }
368
369 if (Import.NrIndices > 0)
370 {
371 Result.Append(Import.NrIndices.ToString());
372 if (Import.NrIndices > 1)
373 Result.AppendLine(" indices.");
374 else
375 Result.AppendLine(" index.");
376 }
377
378 if (Import.NrBlocks > 0)
379 {
380 Result.Append(Import.NrBlocks.ToString());
381 if (Import.NrBlocks > 1)
382 Result.AppendLine(" blocks.");
383 else
384 Result.AppendLine(" block.");
385 }
386
387 if (Import.NrObjects > 0)
388 {
389 Result.Append(Import.NrObjects.ToString());
390 if (Import.NrObjects > 1)
391 Result.AppendLine(" objects.");
392 else
393 Result.AppendLine(" object.");
394 }
395
396 if (Import.NrEntries > 0)
397 {
398 Result.Append(Import.NrEntries.ToString());
399 if (Import.NrEntries > 1)
400 Result.AppendLine(" entries.");
401 else
402 Result.AppendLine(" entry.");
403 }
404
405 if (Import.NrProperties > 0)
406 {
407 Result.Append(Import.NrProperties.ToString());
408 if (Import.NrProperties > 1)
409 Result.AppendLine(" properties.");
410 else
411 Result.AppendLine(" property.");
412 }
413
414 if (Import.NrFiles > 0)
415 {
416 Result.Append(Import.NrFiles.ToString());
417 if (Import.NrFiles > 1)
418 Result.Append(" files");
419 else
420 Result.Append(" file");
421
422 Result.Append(" (");
423 Result.Append(Export.FormatBytes(Import.NrFileBytes));
424 Result.AppendLine(").");
425 }
426
427 if (Import is RestoreBackupFile Restore && Restore.NrObjectsFailed > 0)
428 {
429 Result.Append(Restore.NrObjectsFailed.ToString());
430 if (Import.NrProperties > 1)
431 Result.Append(" objects");
432 else
433 Result.Append(" object");
434
435 Result.AppendLine(" failed.");
436 }
437 }
438
439 if (Overwrite)
440 {
441 Result.AppendLine();
442 Result.Append("Click on the Next button to continue.");
443 }
444
445 await ClientEvents.PushEvent(GetTabIDs(TabID), "RestoreFinished", JSON.Encode(new Dictionary<string, object>()
446 {
447 { "ok", true },
448 { "message", Result.ToString() }
449 }, false), true);
450 }
451 catch (Exception ex)
452 {
453 Log.Exception(ex);
454 ShowStatus(TabID, "Failure: " + ex.Message);
455
456 await ClientEvents.PushEvent(GetTabIDs(TabID), "RestoreFinished", JSON.Encode(new Dictionary<string, object>()
457 {
458 { "ok", false },
459 { "message", ex.Message }
460 }, false), true);
461 }
462 finally
463 {
464 AesTransform1?.Dispose();
465 AesTransform2?.Dispose();
466 cs1?.Dispose();
467 cs2?.Dispose();
468 BackupFile?.Dispose();
469 KeyFile?.Dispose();
470 }
471 }
472
473 private static async Task<(ICryptoTransform, CryptoStream)> DoImport(FileStream BackupFile, FileStream KeyFile, string TabID,
474 string Extension, ValidateBackupFile Import, bool Overwrite, bool OnlySelectedCollections, Array SelectedCollections,
475 Array SelectedParts)
476 {
477 ICryptoTransform AesTransform = null;
478 CryptoStream cs = null;
479
480 BackupFile.Position = 0;
481
482 switch (Extension.ToLower())
483 {
484 case ".xml":
485 await RestoreXml(BackupFile, TabID, Import, Overwrite, OnlySelectedCollections, SelectedCollections, SelectedParts);
486 break;
487
488 case ".bin":
489 await RestoreBinary(BackupFile, TabID, Import, Overwrite, OnlySelectedCollections, SelectedCollections, SelectedParts);
490 break;
491
492 case ".gz":
493 await RestoreCompressed(BackupFile, TabID, Import, Overwrite, OnlySelectedCollections, SelectedCollections, SelectedParts);
494 break;
495
496 case ".bak":
497 if (KeyFile is null)
498 throw new Exception("No key file provided.");
499
500 KeyFile.Position = 0;
501
502 (AesTransform, cs) = await RestoreEncrypted(BackupFile, KeyFile, TabID, Import, Overwrite, OnlySelectedCollections, SelectedCollections, SelectedParts);
503 break;
504
505 default:
506 throw new Exception("Unrecognized file extension: " + Extension);
507 }
508
509 return (AesTransform, cs);
510 }
511
512 private static async Task RestoreXml(Stream BackupFile, string TabID, ValidateBackupFile Import, bool Overwrite,
513 bool OnlySelectedCollections, Array SelectedCollections, Array SelectedParts)
514 {
515 XmlReaderSettings Settings = new XmlReaderSettings()
516 {
517 Async = true,
518 CloseInput = true,
519 ConformanceLevel = ConformanceLevel.Document,
520 CheckCharacters = true,
521 DtdProcessing = DtdProcessing.Prohibit,
522 IgnoreComments = true,
523 IgnoreProcessingInstructions = true,
524 IgnoreWhitespace = true
525 };
526 XmlReader r = XmlReader.Create(BackupFile, Settings);
527 DateTime LastReport = DateTime.Now;
528 KeyValuePair<string, object> P;
529 bool ImportCollection = !OnlySelectedCollections;
530 bool ImportPart = !OnlySelectedCollections;
531 bool DatabaseStarted = false;
532 bool LedgerStarted = false;
533 bool CollectionStarted = false;
534 bool IndexStarted = false;
535 bool BlockStarted = false;
536 bool FilesStarted = false;
537 bool FirstFile = true;
538
539 if (!r.ReadToFollowing("Export", XmlFileLedger.Namespace))
540 throw new Exception("Invalid backup XML file.");
541
542 await Import.Start();
543
544 while (await r.ReadAsync())
545 {
546 if (r.IsStartElement())
547 {
548 switch (r.LocalName)
549 {
550 case "Database":
551 if (r.Depth != 1)
552 throw new Exception("Database element not expected.");
553
554 ImportPart = !OnlySelectedCollections || Array.IndexOf(SelectedParts, "Database") >= 0 || SelectedParts.Length == 0;
555
556 if (!ImportPart)
557 ShowStatus(TabID, "Skipping database section.");
558 else if (Overwrite)
559 ShowStatus(TabID, "Restoring database section.");
560 else
561 ShowStatus(TabID, "Validating database section.");
562
563 await Import.StartDatabase();
564 DatabaseStarted = true;
565 break;
566
567 case "Ledger":
568 if (r.Depth != 1)
569 throw new Exception("Ledger element not expected.");
570
571 ImportPart = !OnlySelectedCollections || Array.IndexOf(SelectedParts, "Ledger") >= 0 || SelectedParts.Length == 0;
572
573 if (!ImportPart)
574 ShowStatus(TabID, "Skipping ledger section.");
575 else if (Overwrite)
576 ShowStatus(TabID, "Restoring ledger section.");
577 else
578 ShowStatus(TabID, "Validating ledger section.");
579
580 await Import.StartLedger();
581 LedgerStarted = true;
582 break;
583
584 case "Collection":
585 if (r.Depth != 2 || (!DatabaseStarted && !LedgerStarted))
586 throw new Exception("Collection element not expected.");
587
588 if (!r.MoveToAttribute("name"))
589 throw new Exception("Collection name missing.");
590
591 string CollectionName = r.Value;
592
593 if (IndexStarted)
594 {
595 await Import.EndIndex();
596 IndexStarted = false;
597 }
598 else if (BlockStarted)
599 {
600 await Import.EndBlock();
601 BlockStarted = false;
602 }
603
604 if (CollectionStarted)
605 await Import.EndCollection();
606
607 if (OnlySelectedCollections)
608 {
609 ImportCollection = ImportPart && (Array.IndexOf(SelectedCollections, CollectionName) >= 0 ||
610 SelectedCollections.Length == 0);
611 }
612
613 if (ImportCollection)
614 {
615 await Import.StartCollection(CollectionName);
616 CollectionStarted = true;
617
618 CollectionFound(TabID, CollectionName);
619 }
620 else
621 CollectionStarted = false;
622 break;
623
624 case "Index":
625 if (r.Depth != 3 || !CollectionStarted)
626 throw new Exception("Index element not expected.");
627
628 if (ImportCollection)
629 {
630 await Import.StartIndex();
631 IndexStarted = true;
632 }
633 break;
634
635 case "Field":
636 if (r.Depth != 4 || !IndexStarted)
637 throw new Exception("Field element not expected.");
638
639 if (r.MoveToFirstAttribute())
640 {
641 string FieldName = null;
642 bool Ascending = true;
643
644 do
645 {
646 switch (r.LocalName)
647 {
648 case "name":
649 FieldName = r.Value;
650 break;
651
652 case "ascending":
653 if (!CommonTypes.TryParse(r.Value, out Ascending))
654 throw new Exception("Invalid boolean value.");
655 break;
656
657 case "xmlns":
658 break;
659
660 default:
661 throw new Exception("Unexpected attribute: " + r.LocalName);
662 }
663 }
664 while (r.MoveToNextAttribute());
665
666 if (string.IsNullOrEmpty(FieldName))
667 throw new Exception("Invalid field name.");
668
669 if (ImportCollection)
670 await Import.ReportIndexField(FieldName, Ascending);
671 }
672 else
673 throw new Exception("Field attributes expected.");
674
675 break;
676
677 case "Obj":
678 if (r.Depth == 3 && CollectionStarted)
679 {
680 if (IndexStarted)
681 {
682 await Import.EndIndex();
683 IndexStarted = false;
684 }
685
686 using (XmlReader r2 = r.ReadSubtree())
687 {
688 await r2.ReadAsync();
689
690 if (!r2.MoveToFirstAttribute())
691 throw new Exception("Object attributes missing.");
692
693 string ObjectId = null;
694 string TypeName = string.Empty;
695
696 do
697 {
698 switch (r2.LocalName)
699 {
700 case "id":
701 ObjectId = r2.Value;
702 break;
703
704 case "type":
705 TypeName = r2.Value;
706 break;
707
708 case "xmlns":
709 break;
710
711 default:
712 throw new Exception("Unexpected attribute: " + r2.LocalName);
713 }
714 }
715 while (r2.MoveToNextAttribute());
716
717 if (ImportCollection)
718 await Import.StartObject(ObjectId, TypeName);
719
720 while (await r2.ReadAsync())
721 {
722 if (r2.IsStartElement())
723 {
724 P = await ReadValue(r2);
725
726 if (ImportCollection)
727 await Import.ReportProperty(P.Key, P.Value);
728 }
729 }
730 }
731
732 if (ImportCollection)
733 await Import.EndObject();
734 }
735 else
736 throw new Exception("Obj element not expected.");
737
738 break;
739
740 case "Block":
741 if (r.Depth != 3 || !CollectionStarted)
742 throw new Exception("Block element not expected.");
743
744 if (!r.MoveToAttribute("id"))
745 throw new Exception("Block ID missing.");
746
747 string BlockID = r.Value;
748
749 if (ImportCollection)
750 {
751 await Import.StartBlock(BlockID);
752 BlockStarted = true;
753 }
754 break;
755
756 case "MetaData":
757 if (r.Depth == 4 && BlockStarted)
758 {
759 using (XmlReader r2 = r.ReadSubtree())
760 {
761 await r2.ReadAsync();
762
763 while (await r2.ReadAsync())
764 {
765 if (r2.IsStartElement())
766 {
767 P = await ReadValue(r2);
768
769 if (ImportCollection)
770 await Import.BlockMetaData(P.Key, P.Value);
771 }
772 }
773 }
774 }
775 else
776 throw new Exception("MetaData element not expected.");
777 break;
778
779 case "New":
780 case "Update":
781 case "Delete":
782 case "Clear":
783 if (r.Depth != 4 || !CollectionStarted || !BlockStarted)
784 throw new Exception("Entry element not expected.");
785
787
788 switch (r.LocalName)
789 {
790 case "New":
791 EntryType = EntryType.New;
792 break;
793
794 case "Update":
795 EntryType = EntryType.Update;
796 break;
797
798 case "Delete":
799 EntryType = EntryType.Delete;
800 break;
801
802 case "Clear":
803 EntryType = EntryType.Clear;
804 break;
805
806 default:
807 throw new Exception("Unexpected element: " + r.LocalName);
808 }
809
810 using (XmlReader r2 = r.ReadSubtree())
811 {
812 await r2.ReadAsync();
813
814 if (!r2.MoveToFirstAttribute())
815 throw new Exception("Object attributes missing.");
816
817 string ObjectId = null;
818 string TypeName = string.Empty;
819 DateTimeOffset EntryTimestamp = DateTimeOffset.MinValue;
820
821 do
822 {
823 switch (r2.LocalName)
824 {
825 case "id":
826 ObjectId = r2.Value;
827 break;
828
829 case "type":
830 TypeName = r2.Value;
831 break;
832
833 case "ts":
834 if (!XML.TryParse(r2.Value, out EntryTimestamp))
835 throw new Exception("Invalid Entry Timestamp: " + r2.Value);
836 break;
837
838 case "xmlns":
839 break;
840
841 default:
842 throw new Exception("Unexpected attribute: " + r2.LocalName);
843 }
844 }
845 while (r2.MoveToNextAttribute());
846
847 if (ImportCollection)
848 await Import.StartEntry(ObjectId, TypeName, EntryType, EntryTimestamp);
849
850 while (await r2.ReadAsync())
851 {
852 if (r2.IsStartElement())
853 {
854 P = await ReadValue(r2);
855
856 if (ImportCollection)
857 await Import.ReportProperty(P.Key, P.Value);
858 }
859 }
860 }
861
862 if (ImportCollection)
863 await Import.EndEntry();
864
865 break;
866
867 case "Files":
868 if (r.Depth != 1)
869 throw new Exception("Files element not expected.");
870
871 ImportPart = !OnlySelectedCollections || Array.IndexOf(SelectedParts, "Files") >= 0 || SelectedParts.Length == 0;
872
873 if (IndexStarted)
874 {
875 await Import.EndIndex();
876 IndexStarted = false;
877 }
878 else if (BlockStarted)
879 {
880 await Import.EndBlock();
881 BlockStarted = false;
882 }
883
884 if (CollectionStarted)
885 {
886 await Import.EndCollection();
887 CollectionStarted = false;
888 }
889
890 if (DatabaseStarted)
891 {
892 await Import.EndDatabase();
893 DatabaseStarted = false;
894 }
895 else if (LedgerStarted)
896 {
897 await Import.EndLedger();
898 LedgerStarted = false;
899 }
900
901 if (!ImportPart)
902 ShowStatus(TabID, "Skipping files section.");
903 else if (Overwrite)
904 ShowStatus(TabID, "Restoring files section.");
905 else
906 ShowStatus(TabID, "Validating files section.");
907
908 await Import.StartFiles();
909 FilesStarted = true;
910 break;
911
912 case "File":
913 if (r.Depth != 2 || !FilesStarted)
914 throw new Exception("File element not expected.");
915
916 using (XmlReader r2 = r.ReadSubtree())
917 {
918 if (ImportPart)
919 {
920 await r2.ReadAsync();
921
922 if (!r2.MoveToAttribute("fileName"))
923 throw new Exception("File name missing.");
924
925 string FileName = r.Value;
926
927 if (Path.IsPathRooted(FileName))
928 {
929 if (FileName.StartsWith(Gateway.AppDataFolder))
930 FileName = FileName.Substring(Gateway.AppDataFolder.Length);
931 else
932 throw new Exception("Absolute path names not allowed: " + FileName);
933 }
934
935 FileName = Path.Combine(Gateway.AppDataFolder, FileName);
936
937 using (TemporaryFile fs = new TemporaryFile())
938 {
939 while (await r2.ReadAsync())
940 {
941 if (r2.IsStartElement())
942 {
943 while (r2.LocalName == "Chunk")
944 {
945 string Base64 = await r2.ReadElementContentAsStringAsync();
946 byte[] Data = Convert.FromBase64String(Base64);
947 fs.Write(Data, 0, Data.Length);
948 }
949 }
950 }
951
952 fs.Position = 0;
953
954 if (!OnlySelectedCollections)
955 {
956 if (FirstFile && FileName.EndsWith(Gateway.GatewayConfigLocalFileName, StringComparison.CurrentCultureIgnoreCase))
957 ImportGatewayConfig(fs);
958 else
959 await Import.ExportFile(FileName, fs);
960 }
961
962 FirstFile = false;
963 }
964 }
965 }
966 break;
967
968 default:
969 throw new Exception("Unexpected element: " + r.LocalName);
970 }
971 }
972
973 ShowReport(TabID, Import, ref LastReport, Overwrite);
974 }
975
976 if (IndexStarted)
977 await Import.EndIndex();
978 else if (BlockStarted)
979 await Import.EndBlock();
980
981 if (CollectionStarted)
982 await Import.EndCollection();
983
984 if (DatabaseStarted)
985 await Import.EndDatabase();
986 else if (LedgerStarted)
987 await Import.EndLedger();
988
989 if (FilesStarted)
990 await Import.EndFiles();
991
992 await Import.End();
993 ShowReport(TabID, Import, Overwrite);
994 }
995
996 private static void ShowReport(string TabID, ValidateBackupFile Import, ref DateTime LastReport, bool Overwrite)
997 {
998 DateTime Now = DateTime.Now;
999 if ((Now - LastReport).TotalSeconds >= 1)
1000 {
1001 LastReport = Now;
1002 ShowReport(TabID, Import, Overwrite);
1003 }
1004 }
1005
1006 private static void ShowReport(string TabID, ValidateBackupFile Import, bool Overwrite)
1007 {
1008 string Suffix = Overwrite ? "2" : "1";
1009
1010 if (Import.NrCollections > 0)
1011 ShowStatus(TabID, "NrCollections" + Suffix, Import.NrCollections.ToString() + " collections.");
1012
1013 if (Import.NrIndices > 0)
1014 ShowStatus(TabID, "NrIndices" + Suffix, Import.NrIndices.ToString() + " indices.");
1015
1016 if (Import.NrBlocks > 0)
1017 ShowStatus(TabID, "NrBlocks" + Suffix, Import.NrBlocks.ToString() + " blocks.");
1018
1019 if (Import.NrObjects > 0)
1020 ShowStatus(TabID, "NrObjects" + Suffix, Import.NrObjects.ToString() + " objects.");
1021
1022 if (Import.NrEntries > 0)
1023 ShowStatus(TabID, "NrEntries" + Suffix, Import.NrEntries.ToString() + " entries.");
1024
1025 if (Import.NrProperties > 0)
1026 ShowStatus(TabID, "NrProperties" + Suffix, Import.NrProperties.ToString() + " properties.");
1027
1028 if (Import.NrFiles > 0)
1029 ShowStatus(TabID, "NrFiles" + Suffix, Import.NrFiles.ToString() + " files (" + Export.FormatBytes(Import.NrFileBytes) + ").");
1030
1031 if (Import is RestoreBackupFile Restore && Restore.NrObjectsFailed > 0)
1032 ShowStatus(TabID, "NrFailed" + Suffix, Restore.NrObjectsFailed.ToString() + " objects failed.");
1033 }
1034
1035 private static async Task<KeyValuePair<string, object>> ReadValue(XmlReader r)
1036 {
1037 string PropertyType = r.LocalName;
1038 bool ReadSubtree = (PropertyType == "Bin" || PropertyType == "Array" || PropertyType == "Obj");
1039
1040 if (ReadSubtree)
1041 {
1042 r = r.ReadSubtree();
1043 await r.ReadAsync();
1044 }
1045
1046 if (!r.MoveToFirstAttribute())
1047 {
1048 if (ReadSubtree)
1049 r.Dispose();
1050
1051 throw new Exception("Property attributes missing.");
1052 }
1053
1054 string ElementType = null;
1055 string PropertyName = null;
1056 object Value = null;
1057
1058 do
1059 {
1060 switch (r.LocalName)
1061 {
1062 case "n":
1063 PropertyName = r.Value;
1064 break;
1065
1066 case "v":
1067 switch (PropertyType)
1068 {
1069 case "S":
1070 case "En":
1071 Value = r.Value;
1072 break;
1073
1074 case "S64":
1075 Value = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(r.Value));
1076 break;
1077
1078 case "Null":
1079 Value = null;
1080 break;
1081
1082 case "Bl":
1083 if (CommonTypes.TryParse(r.Value, out bool bl))
1084 Value = bl;
1085 else
1086 {
1087 if (ReadSubtree)
1088 r.Dispose();
1089
1090 throw new Exception("Invalid boolean value.");
1091 }
1092 break;
1093
1094 case "B":
1095 if (byte.TryParse(r.Value, out byte b))
1096 Value = b;
1097 else
1098 {
1099 if (ReadSubtree)
1100 r.Dispose();
1101
1102 throw new Exception("Invalid byte value.");
1103 }
1104 break;
1105
1106 case "Ch":
1107 string s = r.Value;
1108 if (s.Length == 1)
1109 Value = s[0];
1110 else
1111 {
1112 if (ReadSubtree)
1113 r.Dispose();
1114
1115 throw new Exception("Invalid character value.");
1116 }
1117 break;
1118
1119 case "CIS":
1120 Value = (CaseInsensitiveString)r.Value;
1121 break;
1122
1123 case "CIS64":
1124 Value = (CaseInsensitiveString)System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(r.Value));
1125 break;
1126
1127 case "DT":
1128 if (XML.TryParse(r.Value, out DateTime DT))
1129 Value = DT;
1130 else
1131 {
1132 if (ReadSubtree)
1133 r.Dispose();
1134
1135 throw new Exception("Invalid DateTime value.");
1136 }
1137 break;
1138
1139 case "DTO":
1140 if (XML.TryParse(r.Value, out DateTimeOffset DTO))
1141 Value = DTO;
1142 else
1143 {
1144 if (ReadSubtree)
1145 r.Dispose();
1146
1147 throw new Exception("Invalid DateTimeOffset value.");
1148 }
1149 break;
1150
1151 case "Dc":
1152 if (CommonTypes.TryParse(r.Value, out decimal dc))
1153 Value = dc;
1154 else
1155 {
1156 if (ReadSubtree)
1157 r.Dispose();
1158
1159 throw new Exception("Invalid Decimal value.");
1160 }
1161 break;
1162
1163 case "Db":
1164 if (CommonTypes.TryParse(r.Value, out double db))
1165 Value = db;
1166 else
1167 {
1168 if (ReadSubtree)
1169 r.Dispose();
1170
1171 throw new Exception("Invalid Double value.");
1172 }
1173 break;
1174
1175 case "I2":
1176 if (short.TryParse(r.Value, out short i2))
1177 Value = i2;
1178 else
1179 {
1180 if (ReadSubtree)
1181 r.Dispose();
1182
1183 throw new Exception("Invalid Int16 value.");
1184 }
1185 break;
1186
1187 case "I4":
1188 if (int.TryParse(r.Value, out int i4))
1189 Value = i4;
1190 else
1191 {
1192 if (ReadSubtree)
1193 r.Dispose();
1194
1195 throw new Exception("Invalid Int32 value.");
1196 }
1197 break;
1198
1199 case "I8":
1200 if (long.TryParse(r.Value, out long i8))
1201 Value = i8;
1202 else
1203 {
1204 if (ReadSubtree)
1205 r.Dispose();
1206
1207 throw new Exception("Invalid Int64 value.");
1208 }
1209 break;
1210
1211 case "I1":
1212 if (sbyte.TryParse(r.Value, out sbyte i1))
1213 Value = i1;
1214 else
1215 {
1216 if (ReadSubtree)
1217 r.Dispose();
1218
1219 throw new Exception("Invalid SByte value.");
1220 }
1221 break;
1222
1223 case "Fl":
1224 if (CommonTypes.TryParse(r.Value, out float fl))
1225 Value = fl;
1226 else
1227 {
1228 if (ReadSubtree)
1229 r.Dispose();
1230
1231 throw new Exception("Invalid Single value.");
1232 }
1233 break;
1234
1235 case "U2":
1236 if (ushort.TryParse(r.Value, out ushort u2))
1237 Value = u2;
1238 else
1239 {
1240 if (ReadSubtree)
1241 r.Dispose();
1242
1243 throw new Exception("Invalid UInt16 value.");
1244 }
1245 break;
1246
1247 case "U4":
1248 if (uint.TryParse(r.Value, out uint u4))
1249 Value = u4;
1250 else
1251 {
1252 if (ReadSubtree)
1253 r.Dispose();
1254
1255 throw new Exception("Invalid UInt32 value.");
1256 }
1257 break;
1258
1259 case "U8":
1260 if (ulong.TryParse(r.Value, out ulong u8))
1261 Value = u8;
1262 else
1263 {
1264 if (ReadSubtree)
1265 r.Dispose();
1266
1267 throw new Exception("Invalid UInt64 value.");
1268 }
1269 break;
1270
1271 case "TS":
1272 if (TimeSpan.TryParse(r.Value, out TimeSpan TS))
1273 Value = TS;
1274 else
1275 {
1276 if (ReadSubtree)
1277 r.Dispose();
1278
1279 throw new Exception("Invalid TimeSpan value.");
1280 }
1281 break;
1282
1283 case "Bin":
1284 if (ReadSubtree)
1285 r.Dispose();
1286
1287 throw new Exception("Binary member values are reported using child elements.");
1288
1289 case "ID":
1290 if (Guid.TryParse(r.Value, out Guid Id))
1291 Value = Id;
1292 else
1293 {
1294 if (ReadSubtree)
1295 r.Dispose();
1296
1297 throw new Exception("Invalid GUID value.");
1298 }
1299 break;
1300
1301 case "Array":
1302 if (ReadSubtree)
1303 r.Dispose();
1304
1305 throw new Exception("Arrays report values as child elements.");
1306
1307 case "Obj":
1308 if (ReadSubtree)
1309 r.Dispose();
1310
1311 throw new Exception("Objects report member values as child elements.");
1312
1313 default:
1314 if (ReadSubtree)
1315 r.Dispose();
1316
1317 throw new Exception("Unexpected property type: " + PropertyType);
1318 }
1319 break;
1320
1321 case "elementType":
1322 case "type":
1323 ElementType = r.Value;
1324 break;
1325
1326 case "xmlns":
1327 break;
1328
1329 default:
1330 if (ReadSubtree)
1331 r.Dispose();
1332
1333 throw new Exception("Unexpected attribute: " + r.LocalName);
1334 }
1335 }
1336 while (r.MoveToNextAttribute());
1337
1338 if (!(ElementType is null))
1339 {
1340 switch (PropertyType)
1341 {
1342 case "Array":
1343 List<object> List = new List<object>();
1344
1345 while (await r.ReadAsync())
1346 {
1347 if (r.IsStartElement())
1348 {
1349 KeyValuePair<string, object> P = await ReadValue(r);
1350 if (!string.IsNullOrEmpty(P.Key))
1351 {
1352 if (ReadSubtree)
1353 r.Dispose();
1354
1355 throw new Exception("Arrays do not contain property names.");
1356 }
1357
1358 List.Add(P.Value);
1359 }
1360 else if (r.NodeType == XmlNodeType.EndElement)
1361 break;
1362 }
1363
1364 Value = List.ToArray();
1365 break;
1366
1367 case "Obj":
1368 GenericObject GenObj = new GenericObject(string.Empty, ElementType, Guid.Empty);
1369 Value = GenObj;
1370
1371 while (await r.ReadAsync())
1372 {
1373 if (r.IsStartElement())
1374 {
1375 KeyValuePair<string, object> P = await ReadValue(r);
1376 GenObj[P.Key] = P.Value;
1377 }
1378 else if (r.NodeType == XmlNodeType.EndElement)
1379 break;
1380 }
1381 break;
1382
1383 default:
1384 if (ReadSubtree)
1385 r.Dispose();
1386
1387 throw new Exception("Type only valid option for arrays and objects.");
1388 }
1389 }
1390 else if (PropertyType == "Bin")
1391 {
1392 MemoryStream Bin = new MemoryStream();
1393
1394 while (await r.ReadAsync())
1395 {
1396 if (r.IsStartElement())
1397 {
1398 try
1399 {
1400 while (r.LocalName == "Chunk")
1401 {
1402 string Base64 = await r.ReadElementContentAsStringAsync();
1403 byte[] Data = Convert.FromBase64String(Base64);
1404 Bin.Write(Data, 0, Data.Length);
1405 }
1406 }
1407 catch (Exception ex)
1408 {
1409 if (ReadSubtree)
1410 r.Dispose();
1411
1412 System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(ex).Throw();
1413 }
1414 }
1415 else if (r.NodeType == XmlNodeType.EndElement)
1416 break;
1417 }
1418
1419 Value = Bin.ToArray();
1420 }
1421
1422 if (ReadSubtree)
1423 r.Dispose();
1424
1425 return new KeyValuePair<string, object>(PropertyName, Value);
1426 }
1427
1428 private static async Task RestoreBinary(Stream BackupFile, string TabID, ValidateBackupFile Import, bool Overwrite,
1429 bool OnlySelectedCollections, Array SelectedCollections, Array SelectedParts)
1430 {
1431 int Version = BackupFile.ReadByte();
1432 if (Version != 1)
1433 throw new Exception("File version not supported.");
1434
1435 DateTime LastReport = DateTime.Now;
1436 byte Command;
1437 bool ImportCollection = !OnlySelectedCollections;
1438 bool ImportPart;
1439
1440 using (BinaryReader r = new BinaryReader(BackupFile, System.Text.Encoding.UTF8, true))
1441 {
1442 string s = r.ReadString();
1443 if (s != BinaryExportFormat.Preamble)
1444 throw new Exception("Invalid backup file.");
1445
1446 await Import.Start();
1447
1448 while ((Command = r.ReadByte()) != 0)
1449 {
1450 switch (Command)
1451 {
1452 case 1:
1453 throw new Exception("Obsolete file."); // 1 is obsolete (previously XMPP Credentials)
1454
1455 case 2: // Database
1456 string CollectionName;
1457 string ObjectId;
1458 string TypeName;
1459 string FieldName;
1460 bool Ascending;
1461
1462 ImportPart = !OnlySelectedCollections || Array.IndexOf(SelectedParts, "Database") >= 0 || SelectedParts.Length == 0;
1463
1464 if (!ImportPart)
1465 ShowStatus(TabID, "Skipping database section.");
1466 else if (Overwrite)
1467 ShowStatus(TabID, "Restoring database section.");
1468 else
1469 ShowStatus(TabID, "Validating database section.");
1470
1471 await Import.StartDatabase();
1472
1473 while (!string.IsNullOrEmpty(CollectionName = r.ReadString()))
1474 {
1475 if (OnlySelectedCollections)
1476 {
1477 ImportCollection = ImportPart && (Array.IndexOf(SelectedCollections, CollectionName) >= 0 ||
1478 SelectedCollections.Length == 0);
1479 }
1480
1481 if (ImportCollection)
1482 {
1483 await Import.StartCollection(CollectionName);
1484 CollectionFound(TabID, CollectionName);
1485 }
1486
1487 byte b;
1488
1489 while ((b = r.ReadByte()) != 0)
1490 {
1491 switch (b)
1492 {
1493 case 1:
1494 if (ImportCollection)
1495 await Import.StartIndex();
1496
1497 while (!string.IsNullOrEmpty(FieldName = r.ReadString()))
1498 {
1499 Ascending = r.ReadBoolean();
1500
1501 if (ImportCollection)
1502 await Import.ReportIndexField(FieldName, Ascending);
1503 }
1504
1505 if (ImportCollection)
1506 await Import.EndIndex();
1507 break;
1508
1509 case 2:
1510 ObjectId = r.ReadString();
1511 TypeName = r.ReadString();
1512
1513 if (ImportCollection)
1514 await Import.StartObject(ObjectId, TypeName);
1515
1516 byte PropertyType = r.ReadByte();
1517 string PropertyName = r.ReadString();
1518 object PropertyValue;
1519
1520 while (!string.IsNullOrEmpty(PropertyName))
1521 {
1522 PropertyValue = ReadValue(r, PropertyType);
1523
1524 if (ImportCollection)
1525 await Import.ReportProperty(PropertyName, PropertyValue);
1526
1527 PropertyType = r.ReadByte();
1528 PropertyName = r.ReadString();
1529 }
1530
1531 if (ImportCollection)
1532 await Import.EndObject();
1533 break;
1534
1535 default:
1536 throw new Exception("Unsupported collection section: " + b.ToString());
1537 }
1538
1539 ShowReport(TabID, Import, ref LastReport, Overwrite);
1540 }
1541
1542 if (ImportCollection)
1543 await Import.EndCollection();
1544 }
1545
1546 await Import.EndDatabase();
1547 break;
1548
1549 case 3: // Files
1550 string FileName;
1551 int MaxLen = 256 * 1024;
1552 byte[] Buffer = new byte[MaxLen];
1553
1554 ImportPart = !OnlySelectedCollections || Array.IndexOf(SelectedParts, "Files") >= 0 || SelectedParts.Length == 0;
1555
1556 if (!ImportPart)
1557 ShowStatus(TabID, "Skipping files section.");
1558 else if (Overwrite)
1559 ShowStatus(TabID, "Restoring files section.");
1560 else
1561 ShowStatus(TabID, "Validating files section.");
1562
1563 await Import.StartFiles();
1564
1565 bool FirstFile = true;
1566
1567 while (!string.IsNullOrEmpty(FileName = r.ReadString()))
1568 {
1569 long Length = r.ReadInt64();
1570
1571 if (Path.IsPathRooted(FileName))
1572 {
1573 if (FileName.StartsWith(Gateway.AppDataFolder))
1574 FileName = FileName.Substring(Gateway.AppDataFolder.Length);
1575 else
1576 throw new Exception("Absolute path names not allowed: " + FileName);
1577 }
1578
1579 FileName = Path.Combine(Gateway.AppDataFolder, FileName);
1580
1581 using (TemporaryFile File = new TemporaryFile())
1582 {
1583 while (Length > 0)
1584 {
1585 int Nr = r.Read(Buffer, 0, (int)Math.Min(Length, MaxLen));
1586 Length -= Nr;
1587 await File.WriteAsync(Buffer, 0, Nr);
1588 }
1589
1590 File.Position = 0;
1591 if (ImportPart)
1592 {
1593 try
1594 {
1595 if (FirstFile && FileName.EndsWith(Gateway.GatewayConfigLocalFileName, StringComparison.CurrentCultureIgnoreCase))
1596 ImportGatewayConfig(File);
1597 else
1598 await Import.ExportFile(FileName, File);
1599
1600 ShowReport(TabID, Import, ref LastReport, Overwrite);
1601 }
1602 catch (Exception ex)
1603 {
1604 ShowStatus(TabID, "Unable to restore " + FileName + ": " + ex.Message);
1605 }
1606 }
1607
1608 FirstFile = false;
1609 }
1610 }
1611
1612 await Import.EndFiles();
1613 break;
1614
1615 case 4:
1616 throw new Exception("Export file contains reported errors.");
1617
1618 case 5:
1619 throw new Exception("Export file contains reported exceptions.");
1620
1621 case 6: // Ledger
1622
1623 ImportPart = !OnlySelectedCollections || Array.IndexOf(SelectedParts, "Ledger") >= 0 || SelectedParts.Length == 0;
1624
1625 if (!ImportPart)
1626 ShowStatus(TabID, "Skipping ledger section.");
1627 else if (Overwrite)
1628 ShowStatus(TabID, "Restoring ledger section.");
1629 else
1630 ShowStatus(TabID, "Validating ledger section.");
1631
1632 await Import.StartLedger();
1633
1634 while (!string.IsNullOrEmpty(CollectionName = r.ReadString()))
1635 {
1636 if (OnlySelectedCollections)
1637 {
1638 ImportCollection = ImportPart && (Array.IndexOf(SelectedCollections, CollectionName) >= 0 ||
1639 SelectedCollections.Length == 0);
1640 }
1641
1642 if (ImportCollection)
1643 {
1644 await Import.StartCollection(CollectionName);
1645 CollectionFound(TabID, CollectionName);
1646 }
1647
1648 byte b;
1649
1650 while ((b = r.ReadByte()) != 0)
1651 {
1652 switch (b)
1653 {
1654 case 1:
1655 string BlockID = r.ReadString();
1656 if (ImportCollection)
1657 await Import.StartBlock(BlockID);
1658 break;
1659
1660 case 2:
1661 ObjectId = r.ReadString();
1662 TypeName = r.ReadString();
1663 EntryType EntryType = (EntryType)r.ReadByte();
1664 DateTimeKind Kind = (DateTimeKind)r.ReadByte();
1665 long Ticks = r.ReadInt64();
1666 DateTime DT = new DateTime(Ticks, Kind);
1667 Ticks = r.ReadInt64();
1668 Ticks -= Ticks % 600000000; // Offsets must be in whole minutes.
1669 TimeSpan TS = new TimeSpan(Ticks);
1670 DateTimeOffset EntryTimestamp = new DateTimeOffset(DT, TS);
1671
1672 if (ImportCollection)
1673 await Import.StartEntry(ObjectId, TypeName, EntryType, EntryTimestamp);
1674
1675 byte PropertyType = r.ReadByte();
1676 string PropertyName = r.ReadString();
1677 object PropertyValue;
1678
1679 while (!string.IsNullOrEmpty(PropertyName))
1680 {
1681 PropertyValue = ReadValue(r, PropertyType);
1682
1683 if (ImportCollection)
1684 await Import.ReportProperty(PropertyName, PropertyValue);
1685
1686 PropertyType = r.ReadByte();
1687 PropertyName = r.ReadString();
1688 }
1689
1690 if (ImportCollection)
1691 await Import.EndEntry();
1692 break;
1693
1694 case 3:
1695 if (ImportCollection)
1696 await Import.EndBlock();
1697 break;
1698
1699 case 4:
1700 PropertyName = r.ReadString();
1701 PropertyType = r.ReadByte();
1702 PropertyValue = ReadValue(r, PropertyType);
1703
1704 await Import.BlockMetaData(PropertyName, PropertyValue);
1705 break;
1706
1707 default:
1708 throw new Exception("Unsupported collection section: " + b.ToString());
1709 }
1710
1711 ShowReport(TabID, Import, ref LastReport, Overwrite);
1712 }
1713
1714 if (ImportCollection)
1715 await Import.EndCollection();
1716 }
1717
1718 await Import.EndLedger();
1719 break;
1720
1721 default:
1722 throw new Exception("Unsupported section: " + Command.ToString());
1723 }
1724 }
1725
1726 await Import.End();
1727 ShowReport(TabID, Import, Overwrite);
1728 }
1729 }
1730
1731 private static void ImportGatewayConfig(Stream File)
1732 {
1733 XmlDocument Doc = new XmlDocument()
1734 {
1735 PreserveWhitespace = true
1736 };
1737 Doc.Load(File);
1738
1739 string OriginalFileName = Gateway.ConfigFilePath;
1740 XmlDocument Original = new XmlDocument()
1741 {
1742 PreserveWhitespace = true
1743 };
1744 Original.Load(OriginalFileName);
1745
1746 if (!(Doc.DocumentElement is null) && Doc.DocumentElement.LocalName == "GatewayConfiguration")
1747 {
1748 List<KeyValuePair<string, string>> DefaultPages = null;
1749
1750 foreach (XmlNode N in Doc.DocumentElement.ChildNodes)
1751 {
1752 if (N is XmlElement E)
1753 {
1754 switch (E.LocalName)
1755 {
1756 case "ApplicationName":
1757 string s = E.InnerText;
1758 Gateway.ApplicationName = s;
1759 Original.DocumentElement["ApplicationName"].InnerText = s;
1760 break;
1761
1762 case "DefaultPage":
1763 s = E.InnerText;
1764 string Host = XML.Attribute(E, "host");
1765
1766 if (DefaultPages is null)
1767 DefaultPages = new List<KeyValuePair<string, string>>();
1768
1769 DefaultPages.Add(new KeyValuePair<string, string>(Host, s));
1770 break;
1771
1772 // TODO: Ports ?
1773 // TODO: FileFolders ?
1774 }
1775 }
1776 }
1777
1778 if (!(DefaultPages is null))
1779 {
1780 Gateway.SetDefaultPages(DefaultPages.ToArray());
1781
1782 foreach (XmlNode N in Original.DocumentElement.ChildNodes)
1783 {
1784 if (N is XmlElement E && E.LocalName == "DefaultPage")
1785 {
1786 string Host = XML.Attribute(E, "host");
1787 if (Gateway.TryGetDefaultPage(Host, out string DefaultPage))
1788 E.InnerText = DefaultPage;
1789 }
1790 }
1791 }
1792 }
1793
1794 Original.Save(OriginalFileName);
1795 }
1796
1797 private static object ReadValue(BinaryReader r, byte PropertyType)
1798 {
1799 switch (PropertyType)
1800 {
1801 case BinaryExportFormat.TYPE_BOOLEAN: return r.ReadBoolean();
1802 case BinaryExportFormat.TYPE_BYTE: return r.ReadByte();
1803 case BinaryExportFormat.TYPE_INT16: return r.ReadInt16();
1804 case BinaryExportFormat.TYPE_INT32: return r.ReadInt32();
1805 case BinaryExportFormat.TYPE_INT64: return r.ReadInt64();
1806 case BinaryExportFormat.TYPE_SBYTE: return r.ReadSByte();
1807 case BinaryExportFormat.TYPE_UINT16: return r.ReadUInt16();
1808 case BinaryExportFormat.TYPE_UINT32: return r.ReadUInt32();
1809 case BinaryExportFormat.TYPE_UINT64: return r.ReadUInt64();
1810 case BinaryExportFormat.TYPE_DECIMAL: return r.ReadDecimal();
1811 case BinaryExportFormat.TYPE_DOUBLE: return r.ReadDouble();
1812 case BinaryExportFormat.TYPE_SINGLE: return r.ReadSingle();
1813 case BinaryExportFormat.TYPE_CHAR: return r.ReadChar();
1814 case BinaryExportFormat.TYPE_STRING: return r.ReadString();
1815 case BinaryExportFormat.TYPE_ENUM: return r.ReadString();
1816 case BinaryExportFormat.TYPE_NULL: return null;
1817
1819 DateTimeKind Kind = (DateTimeKind)((int)r.ReadByte());
1820 long Ticks = r.ReadInt64();
1821 return new DateTime(Ticks, Kind);
1822
1824 Ticks = r.ReadInt64();
1825 return new TimeSpan(Ticks);
1826
1828 int Count = r.ReadInt32();
1829 return r.ReadBytes(Count);
1830
1832 byte[] Bin = r.ReadBytes(16);
1833 return new Guid(Bin);
1834
1836 Kind = (DateTimeKind)((int)r.ReadByte());
1837 Ticks = r.ReadInt64();
1838 DateTime DT = new DateTime(Ticks, Kind);
1839 Ticks = r.ReadInt64();
1840 Ticks -= Ticks % 600000000; // Offsets must be in whole minutes.
1841 TimeSpan TS = new TimeSpan(Ticks);
1842 return new DateTimeOffset(DT, TS);
1843
1845 return (CaseInsensitiveString)r.ReadString();
1846
1848 r.ReadString(); // Type name
1849 long NrElements = r.ReadInt64();
1850
1851 List<object> List = new List<object>();
1852
1853 while (NrElements > 0)
1854 {
1855 NrElements--;
1856 PropertyType = r.ReadByte();
1857 List.Add(ReadValue(r, PropertyType));
1858 }
1859
1860 return List.ToArray();
1861
1863 string TypeName = r.ReadString();
1864 GenericObject Object = new GenericObject(string.Empty, TypeName, Guid.Empty);
1865
1866 PropertyType = r.ReadByte();
1867 string PropertyName = r.ReadString();
1868
1869 while (!string.IsNullOrEmpty(PropertyName))
1870 {
1871 Object[PropertyName] = ReadValue(r, PropertyType);
1872
1873 PropertyType = r.ReadByte();
1874 PropertyName = r.ReadString();
1875 }
1876
1877 return Object;
1878
1879 default:
1880 throw new Exception("Unsupported property type: " + PropertyType.ToString());
1881 }
1882 }
1883
1884 private static async Task RestoreCompressed(Stream BackupFile, string TabID, ValidateBackupFile Import, bool Overwrite,
1885 bool OnlySelectedCollections, Array SelectedCollections, Array SelectedParts)
1886 {
1887 using (GZipStream gz = new GZipStream(BackupFile, CompressionMode.Decompress, true))
1888 {
1889 await RestoreBinary(gz, TabID, Import, Overwrite, OnlySelectedCollections, SelectedCollections, SelectedParts);
1890 }
1891 }
1892
1893 private static async Task<(ICryptoTransform, CryptoStream)> RestoreEncrypted(Stream BackupFile, Stream KeyFile, string TabID, ValidateBackupFile Import,
1894 bool Overwrite, bool OnlySelectedCollections, Array SelectedCollections, Array SelectedParts)
1895 {
1896 XmlDocument Doc = new XmlDocument()
1897 {
1898 PreserveWhitespace = true
1899 };
1900
1901 try
1902 {
1903 Doc.Load(KeyFile);
1904 }
1905 catch (Exception)
1906 {
1907 throw new Exception("Invalid key file.");
1908 }
1909
1910 XmlElement KeyAes256 = Doc.DocumentElement;
1911 if (KeyAes256.LocalName != "KeyAes256" ||
1912 KeyAes256.NamespaceURI != XmlFileLedger.Namespace ||
1913 !KeyAes256.HasAttribute("key") ||
1914 !KeyAes256.HasAttribute("iv"))
1915 {
1916 throw new Exception("Invalid key file.");
1917 }
1918
1919 byte[] Key = Convert.FromBase64String(KeyAes256.Attributes["key"].Value);
1920 byte[] IV = Convert.FromBase64String(KeyAes256.Attributes["iv"].Value);
1921
1922 ICryptoTransform AesTransform = WebResources.StartExport.aes.CreateDecryptor(Key, IV);
1923 CryptoStream cs = new CryptoStream(BackupFile, AesTransform, CryptoStreamMode.Read);
1924
1925 await RestoreCompressed(cs, TabID, Import, Overwrite, OnlySelectedCollections, SelectedCollections, SelectedParts);
1926
1927 return (AesTransform, cs);
1928 }
1929
1930 private static async Task DoAnalyze(string TabID)
1931 {
1932 ShowStatus(TabID, "Analyzing database.");
1933
1934 string XmlPath = Path.Combine(Gateway.AppDataFolder, "Restore.xml");
1935 string HtmlPath = Path.Combine(Gateway.AppDataFolder, "Restore.html");
1936 string XsltPath = Path.Combine(Gateway.AppDataFolder, "Transforms", "DbStatXmlToHtml.xslt");
1937 using (FileStream fs = File.Create(XmlPath))
1938 {
1939 XmlWriterSettings Settings = XML.WriterSettings(true, false);
1940 using (XmlWriter w = XmlWriter.Create(fs, Settings))
1941 {
1942 await Database.Analyze(w, XsltPath, Gateway.AppDataFolder, false);
1943 w.Flush();
1944 fs.Flush();
1945 }
1946 }
1947
1948 XslCompiledTransform Xslt = XSL.LoadTransform(typeof(Gateway).Namespace + ".Transforms.DbStatXmlToHtml.xslt");
1949
1950 string s = await Resources.ReadAllTextAsync(XmlPath);
1951 s = XSL.Transform(s, Xslt);
1952 byte[] Bin = WebResources.StartAnalyze.utf8Bom.GetBytes(s);
1953
1954 await Resources.WriteAllBytesAsync(HtmlPath, Bin);
1955
1956 ShowStatus(TabID, "Database analysis successfully completed.");
1957
1958 /*
1959 int i = s.IndexOf("<body>");
1960 if (i > 0)
1961 s = s.Substring(i + 6);
1962
1963 i = s.IndexOf("</body>");
1964 if (i > 0)
1965 s = s.Substring(0, i);
1966
1967 ClientEvents.PushEvent(GetTabIDs(TabID), "ShowStatus", JSON.Encode(new Dictionary<string, object>()
1968 {
1969 { "html", s },
1970 }, false), true);
1971 */
1972 }
1973
1977 public override Task MakeCompleted()
1978 {
1979 return this.MakeCompleted(this.reloadConfiguration);
1980 }
1981
1986 public override Task<bool> SimplifiedConfiguration()
1987 {
1988 return Task.FromResult(true);
1989 }
1990
1994 public const string GATEWAY_RESTORE = nameof(GATEWAY_RESTORE);
1995
1999 public const string GATEWAY_RESTORE_BAKFILE = nameof(GATEWAY_RESTORE_BAKFILE);
2000
2004 public const string GATEWAY_RESTORE_KEYFILE = nameof(GATEWAY_RESTORE_KEYFILE);
2005
2009 public const string GATEWAY_RESTORE_OVERWRITE = nameof(GATEWAY_RESTORE_OVERWRITE);
2010
2014 public const string GATEWAY_RESTORE_COLLECTIONS = nameof(GATEWAY_RESTORE_COLLECTIONS);
2015
2019 public const string GATEWAY_RESTORE_PARTS = nameof(GATEWAY_RESTORE_PARTS);
2020
2025 public override async Task<bool> EnvironmentConfiguration()
2026 {
2027 if (!this.TryGetEnvironmentVariable(GATEWAY_RESTORE, false, out bool Restore))
2028 return false;
2029
2030 if (!Restore)
2031 return true;
2032
2033 if (!this.TryGetEnvironmentVariable(GATEWAY_RESTORE_BAKFILE, true, out string BakFileName) ||
2034 !this.TryGetEnvironmentVariable(GATEWAY_RESTORE_OVERWRITE, true, out bool OverWrite))
2035 {
2036 return false;
2037 }
2038
2039 string KeyFileName = Environment.GetEnvironmentVariable(GATEWAY_RESTORE_KEYFILE);
2040 string CollectionsStr = Environment.GetEnvironmentVariable(GATEWAY_RESTORE_COLLECTIONS);
2041 string PartsStr = Environment.GetEnvironmentVariable(GATEWAY_RESTORE_PARTS);
2042 string[] Collections;
2043 string[] Parts;
2044
2045 if (string.IsNullOrEmpty(CollectionsStr))
2046 Collections = new string[0];
2047 else
2048 Collections = CollectionsStr.Split(',');
2049
2050 if (string.IsNullOrEmpty(PartsStr))
2051 Parts = new string[0];
2052 else
2053 Parts = PartsStr.Split(',');
2054
2055 FileStream BakFile = null;
2056 FileStream KeyFile = null;
2057
2058 try
2059 {
2060 try
2061 {
2062 BakFile = File.OpenRead(BakFileName);
2063 }
2064 catch (Exception ex)
2065 {
2066 this.LogEnvironmentError(ex.Message, GATEWAY_RESTORE_BAKFILE, BakFileName);
2067 return false;
2068 }
2069
2070 if (!string.IsNullOrEmpty(KeyFileName))
2071 {
2072 try
2073 {
2074 KeyFile = File.OpenRead(KeyFileName);
2075 }
2076 catch (Exception ex)
2077 {
2078 this.LogEnvironmentError(ex.Message, GATEWAY_RESTORE_KEYFILE, KeyFileName);
2079 return false;
2080 }
2081 }
2082
2083 await this.Restore(BakFile, KeyFile, string.Empty, BakFileName, OverWrite, Collections.Length > 0, Collections, Parts);
2084 }
2085 catch (Exception ex)
2086 {
2087 this.LogEnvironmentError(ex.Message, GATEWAY_RESTORE_BAKFILE, BakFileName);
2088 return false;
2089 }
2090 finally
2091 {
2092 BakFile?.Dispose();
2093 KeyFile?.Dispose();
2094 }
2095
2096 return true;
2097 }
2098
2099 }
2100}
Helps with parsing of commong data types.
Definition: CommonTypes.cs:13
static bool TryParse(string s, out double Value)
Tries to decode a string encoded double.
Definition: CommonTypes.cs:46
Helps with common JSON-related tasks.
Definition: JSON.cs:14
static string Encode(string s)
Encodes a string for inclusion in JSON.
Definition: JSON.cs:507
Static class managing loading of resources stored as embedded resources or in content files.
Definition: Resources.cs:15
static Task WriteAllBytesAsync(string FileName, byte[] Data)
Creates a binary file asynchronously.
Definition: Resources.cs:216
static async Task< string > ReadAllTextAsync(string FileName)
Reads a text file asynchronously.
Definition: Resources.cs:205
Helps with common XML-related tasks.
Definition: XML.cs:19
static string Attribute(XmlElement E, string Name)
Gets the value of an XML attribute.
Definition: XML.cs:914
static bool TryParse(string s, out DateTime Value)
Tries to decode a string encoded DateTime.
Definition: XML.cs:744
static XmlWriterSettings WriterSettings(bool Indent, bool OmitXmlDeclaration)
Gets an XML writer settings object.
Definition: XML.cs:1177
Static class managing loading of XSL resources stored as embedded resources or in content files.
Definition: XSL.cs:15
static XslCompiledTransform LoadTransform(string ResourceName)
Loads an XSL transformation from an embedded resource.
Definition: XSL.cs:70
static string Transform(string XML, XslCompiledTransform Transform)
Transforms an XML document using an XSL transform.
Definition: XSL.cs:162
Static class managing the application event log. Applications and services log events on this static ...
Definition: Log.cs:13
static void Exception(Exception Exception, string Object, string Actor, string EventId, EventLevel Level, string Facility, string Module, params KeyValuePair< string, object >[] Tags)
Logs an exception. Event type will be determined by the severity of the exception.
Definition: Log.cs:1647
The ClientEvents class allows applications to push information asynchronously to web clients connecte...
Definition: ClientEvents.cs:51
static Task< int > PushEvent(string[] TabIDs, string Type, object Data)
Puses an event to a set of Tabs, given their Tab IDs.
static string[] GetTabIDs()
Gets all open Tab IDs.
Static class managing data export.
Definition: Export.cs:19
static string FormatBytes(double Bytes)
Formats a file size using appropriate unit.
Definition: Export.cs:126
Static class managing the runtime environment of the IoT Gateway.
Definition: Gateway.cs:126
static IUser AssertUserAuthenticated(HttpRequest Request, string Privilege)
Makes sure a request is being made from a session with a successful user login.
Definition: Gateway.cs:3041
static string ConfigFilePath
Full path to Gateway.config file.
Definition: Gateway.cs:2413
static bool TryGetDefaultPage(HttpRequest Request, out string DefaultPage)
Tries to get the default page of a host.
Definition: Gateway.cs:2473
static string AppDataFolder
Application data folder.
Definition: Gateway.cs:2369
const string GatewayConfigLocalFileName
Gateway.config
Definition: Gateway.cs:130
override Task ConfigureSystem()
Is called during startup to configure the system.
override Task UnregisterSetup(HttpServer WebServer)
Unregisters the setup object.
override Task InitSetup(HttpServer WebServer)
Initializes the setup object.
override async Task< bool > EnvironmentConfiguration()
Environment configuration by configuring values available in environment variables.
override Task MakeCompleted()
Sets the configuration task as completed.
override Task< bool > SimplifiedConfiguration()
Simplified configuration by configuring simple default values.
override Task< string > Title(Language Language)
Gets a title for the system configuration.
static RestoreConfiguration Instance
Current instance of configuration.
override string ConfigPrivilege
Minimum required privilege for a user to be allowed to change the configuration defined by the class.
override string Resource
Resource to be redirected to, to perform the configuration.
override void SetStaticInstance(ISystemConfiguration Configuration)
Sets the static instance of the configuration.
override int Priority
Priority of the setting. Configurations are sorted in ascending order.
Abstract base class for system configurations.
const byte TYPE_CI_STRING
Represents a CaseInsensitiveString
The request could not be understood by the server due to malformed syntax. The client SHOULD NOT repe...
Base class for all HTTP fields.
Definition: HttpField.cs:7
bool TryGetHeaderField(string FieldName, out HttpField Field)
Tries to get a named header field.
Definition: HttpHeader.cs:227
Represents an HTTP request.
Definition: HttpRequest.cs:18
Stream DataStream
Data stream, if data is available, or null if data is not available.
Definition: HttpRequest.cs:139
HttpRequestHeader Header
Request header.
Definition: HttpRequest.cs:134
bool HasData
If the request has data.
Definition: HttpRequest.cs:74
Variables Session
Contains session states, if the resource requires sessions, or null otherwise.
Definition: HttpRequest.cs:164
async Task< object > DecodeDataAsync()
Decodes data sent in request.
Definition: HttpRequest.cs:95
Base class for all HTTP resources.
Definition: HttpResource.cs:23
static string GetSessionId(HttpRequest Request, HttpResponse Response)
Gets the session ID used for a request.
Represets a response of an HTTP client request.
Definition: HttpResponse.cs:21
Implements an HTTP server.
Definition: HttpServer.cs:36
HttpResource Register(HttpResource Resource)
Registers a resource with the server.
Definition: HttpServer.cs:1287
bool Unregister(HttpResource Resource)
Unregisters a resource from the server.
Definition: HttpServer.cs:1438
Represents a case-insensitive string.
Static interface for database persistence. In order to work, a database provider has to be assigned t...
Definition: Database.cs:19
static Task< string[]> Analyze(XmlWriter Output, string XsltPath, string ProgramDataFolder, bool ExportData)
Analyzes the database and exports findings to XML.
Definition: Database.cs:1237
Generic object. Contains a sequence of properties.
Simple ledger that records anything that happens in the database to XML files in the program data fol...
const string Namespace
http://waher.se/Schema/Export.xsd
Event arguments for cache item removal events.
KeyType Key
Key of item that was removed.
Repository of all active caches.
Definition: Caches.cs:11
static void ClearAll()
Clears all active caches.
Definition: Caches.cs:65
Contains information about a language.
Definition: Language.cs:17
Task< string > GetStringAsync(Type Type, int Id, string Default)
Gets the string value of a string ID. If no such string exists, a string is created with the default ...
Definition: Language.cs:209
Class managing the contents of a temporary file. When the class is disposed, the temporary file is de...
override void Dispose(bool disposing)
Disposes of the object, and deletes the temporary file.
Interface for system configurations. The gateway will scan all module for system configuration classe...
PropertyType
Type of indexed property.
EntryType
Ledger entry type.
Definition: ILedgerEntry.cs:9