From 703f1604b97560e832830b57e5ba33dd3b8b8937 Mon Sep 17 00:00:00 2001
From: Klaus Fischer <klaus.fischer@eloware.com>
Date: Thu, 10 Aug 2023 21:31:42 +0200
Subject: [PATCH] feature: split large files
---
.reuse/dep5 | 2 +-
Examples/ConsoleAppExample/.gitignore | 1 +
.../Attachments/metadata.json | 101 ++++++++++++++++++
.../ConsoleAppExample.csproj | 7 ++
Examples/ConsoleAppExample/Program.cs | 2 +-
Examples/ConsoleAppExample/SenderDemo.cs | 2 +
.../FitConnect.LargeAttachment.csproj | 13 +++
FitConnect.LargeAttachment/LargeAttachment.cs | 61 +++++++++++
FitConnect.sln | 6 ++
FitConnect/Models/Attachment.cs | 5 +-
FitConnect/Sender.cs | 36 +------
.../Services/Models/v1/Api/MetadataDto.cs | 6 +-
FitConnect/Services/RestCallService.cs | 4 +-
Tests/BasicUnitTest/BasicUnitTest.csproj | 1 +
14 files changed, 204 insertions(+), 43 deletions(-)
create mode 100644 Examples/ConsoleAppExample/Attachments/metadata.json
create mode 100644 FitConnect.LargeAttachment/FitConnect.LargeAttachment.csproj
create mode 100644 FitConnect.LargeAttachment/LargeAttachment.cs
diff --git a/.reuse/dep5 b/.reuse/dep5
index 75b5e82a..87ed7609 100644
--- a/.reuse/dep5
+++ b/.reuse/dep5
@@ -1,6 +1,6 @@
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Files: FitConnect/* Tests/* Examples/*
+Files: FitConnect/* Tests/* Examples/* FitConnect.LargeAttachment/*
FitConnect.sln test.Dockerfile Test.pdf version.sh FitConnect.sln.DotSettings
README.md CHANGELOG.md *.md Dockerfile nuget_upload.sh ConsoleAppExample/* global.json
DummyServerForHeaderTests/* clean_repo.sh
diff --git a/Examples/ConsoleAppExample/.gitignore b/Examples/ConsoleAppExample/.gitignore
index d9ba1b20..82642dc4 100644
--- a/Examples/ConsoleAppExample/.gitignore
+++ b/Examples/ConsoleAppExample/.gitignore
@@ -1,2 +1,3 @@
appsettings.json
encryptionKeys/
+Attachments/OfficeSetup.exe
diff --git a/Examples/ConsoleAppExample/Attachments/metadata.json b/Examples/ConsoleAppExample/Attachments/metadata.json
new file mode 100644
index 00000000..dd8d9bfc
--- /dev/null
+++ b/Examples/ConsoleAppExample/Attachments/metadata.json
@@ -0,0 +1,101 @@
+{
+ "$schema": "https://schema.fitko.de/fit-connect/metadata/1.0.0/metadata.schema.json",
+ "authenticationInformation": [
+ {
+ "content": "12345",
+ "type": "identificationReport",
+ "version": "1.3.5"
+ }
+ ],
+ "contentStructure": {
+ "attachments": [
+ {
+ "attachmentId": "18c79c4e-cbb5-4c28-a3ee-a3c992562c36",
+ "description": "Test Attachment",
+ "filename": "Test.pdf",
+ "hash": {
+ "content": "8b1042900c2039f65fe6c4cb1bca31e2a7a04b61d3ca7d9ae9fc4077068b82ad5512fa298385b025db70551113b762064444b87737e45e657a71be5b88b06e59",
+ "type": "sha512"
+ },
+ "mimeType": "application/pdf",
+ "purpose": "attachment"
+ },
+ {
+ "attachmentId": "8fbce5dd-064b-4461-b7f4-060259e7776a",
+ "description": "Test Attachment #2",
+ "filename": "Test2.pdf",
+ "hash": {
+ "content": "8b1042900c2039f65fe6c4cb1bca31e2a7a04b61d3ca7d9ae9fc4077068b82ad5512fa298385b025db70551113b762064444b87737e45e657a71be5b88b06e59",
+ "type": "sha512"
+ },
+ "mimeType": "application/pdf",
+ "purpose": "attachment"
+ },
+ {
+ "attachmentId": "d9feba27-2799-4e59-89b3-14b05526173f",
+ "description": "",
+ "filename": "data.json",
+ "hash": {
+ "content": "8a35106639b0a4abf8c1933f17f567b502fb33c60519e45cc6e7f0fcd97670838a13e406d0faf27a0e9c039d004c205de4750f1c28d640a2f663f4a0148cea34",
+ "type": "sha512"
+ },
+ "mimeType": "application/json",
+ "purpose": "attachment"
+ },
+ {
+ "attachmentId": "8a2646bf-1ef2-43e6-8b30-ef70873eb2dd",
+ "description": "Just a big file",
+ "filename": "OfficeSetup.exe",
+ "hash": {
+ "content": "46e801d5789b4128f63207360ec4a604e1b9bfc1ac45639a3bc8673f7d4cb2fc75c5b4696d2b7964351d426c3f2dd69227e20843e2cd3b52dd4276ba8c0843e5",
+ "type": "sha512"
+ },
+ "mimeType": "application/pdf",
+ "purpose": "attachment"
+ },
+ {
+ "attachmentId": "a1ec355f-c100-4eca-8d14-89810aeeae40",
+ "description": "The part #1 of the file with the id 8a2646bf-1ef2-43e6-8b30-ef70873eb2dd",
+ "filename": "8a2646bf-1ef2-43e6-8b30-ef70873eb2dd.part1",
+ "hash": {
+ "content": "7adec1ee35cf5ca9555bf4c0f6f322462e25d8627acbebcf06399e2bb70471a407549cc87e81d0c6a36b35d9604c92a7cebaf6b631a07ca246e475e12dcc2f51",
+ "type": "sha512"
+ },
+ "mimeType": "application/pdf",
+ "purpose": "attachment"
+ },
+ {
+ "attachmentId": "142b0c70-7fce-4001-8c62-27f9f5b14f8a",
+ "description": "The part #2 of the file with the id 8a2646bf-1ef2-43e6-8b30-ef70873eb2dd",
+ "filename": "8a2646bf-1ef2-43e6-8b30-ef70873eb2dd.part2",
+ "hash": {
+ "content": "80b558e3764bd4ae772b70ef452b8ea5fe05db559753ce983ce992f4c47cccc78bdba67b225e331108169f16c7e337d436d3f5399298f7d3dffe5b2b9bd9e3e7",
+ "type": "sha512"
+ },
+ "mimeType": "application/pdf",
+ "purpose": "attachment"
+ }
+ ],
+ "data": {
+ "hash": {
+ "content": "8a35106639b0a4abf8c1933f17f567b502fb33c60519e45cc6e7f0fcd97670838a13e406d0faf27a0e9c039d004c205de4750f1c28d640a2f663f4a0148cea34",
+ "type": "sha512"
+ },
+ "submissionSchema": {
+ "mimeType": "application/json",
+ "schemaUri": "https://eloware.dev/projects/fit-connect/schemas/simpleSchema.json"
+ }
+ }
+ },
+ "paymentInformation": {
+ "paymentMethod": "CREDITCARD",
+ "status": "BOOKED",
+ "transactionId": "12345445",
+ "transactionReference": "2345566"
+ },
+ "replyChannel": {
+ "eMail": {
+ "address": "a@example.com"
+ }
+ }
+}
\ No newline at end of file
diff --git a/Examples/ConsoleAppExample/ConsoleAppExample.csproj b/Examples/ConsoleAppExample/ConsoleAppExample.csproj
index 54cd4fda..84462ca0 100644
--- a/Examples/ConsoleAppExample/ConsoleAppExample.csproj
+++ b/Examples/ConsoleAppExample/ConsoleAppExample.csproj
@@ -50,9 +50,16 @@
<None Update="encryptionKeys\set-public-keys.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
+ <None Update="Attachments\zoomusInstallerFull.pkg">
+ <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ </None>
+ <None Update="Attachments\OfficeSetup.exe">
+ <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ </None>
</ItemGroup>
<ItemGroup>
+ <ProjectReference Include="..\..\FitConnect.LargeAttachment\FitConnect.LargeAttachment.csproj" />
<ProjectReference Include="..\..\FitConnect\FitConnect.csproj" />
</ItemGroup>
diff --git a/Examples/ConsoleAppExample/Program.cs b/Examples/ConsoleAppExample/Program.cs
index 9c692a33..74bb4980 100644
--- a/Examples/ConsoleAppExample/Program.cs
+++ b/Examples/ConsoleAppExample/Program.cs
@@ -17,7 +17,7 @@ var config = new ConfigurationBuilder()
var logger = LoggerFactory.Create(
builder => {
builder.AddSimpleConsole();
- builder.SetMinimumLevel(LogLevel.Warning);
+ builder.SetMinimumLevel(LogLevel.Trace);
}).CreateLogger("FIT-Connect");
#endregion
diff --git a/Examples/ConsoleAppExample/SenderDemo.cs b/Examples/ConsoleAppExample/SenderDemo.cs
index 4d0a8210..b40e66c4 100644
--- a/Examples/ConsoleAppExample/SenderDemo.cs
+++ b/Examples/ConsoleAppExample/SenderDemo.cs
@@ -2,6 +2,7 @@ using System.Net.Mime;
using System.Text;
using FitConnect;
using FitConnect.Encryption;
+using FitConnect.LargeAttachment;
using FitConnect.Models;
using FitConnect.Models.Api.Metadata;
using FitConnect.Models.Api.Set;
@@ -43,6 +44,7 @@ public static class SenderDemo {
Attachment.FromString("{\"message\":\"Hello World\"}",
MediaTypeNames.Application.Json, "data.json")
)
+ .AddAttachments(await (new LargeAttachment("./Attachments/OfficeSetup.exe", "application/pdf", "Just a big file").LoadAttachment()))
.SetAuthenticationInformation(new AuthenticationInformation("12345",
AuthenticationInformationType.IdentificationReport, "1.3.5"))
.SetPaymentInformation(new PaymentInformation(PaymentMethod.Creditcard,
diff --git a/FitConnect.LargeAttachment/FitConnect.LargeAttachment.csproj b/FitConnect.LargeAttachment/FitConnect.LargeAttachment.csproj
new file mode 100644
index 00000000..f990f529
--- /dev/null
+++ b/FitConnect.LargeAttachment/FitConnect.LargeAttachment.csproj
@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <TargetFramework>net6.0</TargetFramework>
+ <ImplicitUsings>enable</ImplicitUsings>
+ <Nullable>enable</Nullable>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <ProjectReference Include="..\FitConnect\FitConnect.csproj" />
+ </ItemGroup>
+
+</Project>
diff --git a/FitConnect.LargeAttachment/LargeAttachment.cs b/FitConnect.LargeAttachment/LargeAttachment.cs
new file mode 100644
index 00000000..953d12bf
--- /dev/null
+++ b/FitConnect.LargeAttachment/LargeAttachment.cs
@@ -0,0 +1,61 @@
+using FitConnect.Models;
+using Microsoft.AspNetCore.Http;
+
+namespace FitConnect.LargeAttachment;
+
+public class LargeAttachment {
+ private const int MaxAttachmentSize = 3 * 1024 * 1024;
+ private string FullPath { get; }
+
+ public LargeAttachment(string fileName, string mimeType, string description) {
+ FullPath = fileName;
+ Filename = fileName;
+ MimeType = mimeType;
+ Description = description;
+ }
+
+ public string Description { get; set; }
+
+ public string MimeType { get; set; }
+
+ public string Filename { get; set; }
+
+ /// <summary>
+ /// Loads the attachment from the file system and returns a list of attachments with less than 300MB each
+ /// </summary>
+ /// <returns></returns>
+ public async Task<List<Attachment>> LoadAttachment() {
+ Guid? id = null;
+ var result = new List<Attachment>();
+ var list = await LoadFileAsync();
+ for (var index = 0; index < list.Count; index++) {
+ var chunk = list[index];
+ var filename = index == 0 ? Filename : id.ToString() + ".part" + index;
+ result.Add(Attachment.FromByteArray(chunk, MimeType, filename,
+ index == 0 ? Description : $"The part #{index} of the file with the id {id}"));
+ id ??= result[0].Id;
+ }
+
+ return result;
+ }
+
+ /// <summary>
+ /// Load the file and split it into byte arrays with less than 300MB each if the source file is larger than 300MB
+ /// </summary>
+ public async Task<List<byte[]>> LoadFileAsync() {
+ var fileBytes = new List<byte[]>();
+ var bytesRead = 0;
+ var fileStream = File.Open(FullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+ do {
+ var buffer = new byte[MaxAttachmentSize];
+ bytesRead = await fileStream.ReadAsync(buffer, 0, MaxAttachmentSize);
+ if (bytesRead > 0) {
+ var copy = new byte[bytesRead];
+ Array.Copy(buffer, copy, bytesRead);
+ fileBytes.Add(copy);
+ }
+ } while (bytesRead > 0);
+
+ return fileBytes;
+ }
+}
diff --git a/FitConnect.sln b/FitConnect.sln
index 9c7ca599..97cb550a 100644
--- a/FitConnect.sln
+++ b/FitConnect.sln
@@ -22,6 +22,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests\DummyServerForHeaderT
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ValidationTests", "Tests\ValidationTests\ValidationTests.csproj", "{DDEFDA59-FD75-4F25-9BA2-8DAE22B4A0B0}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FitConnect.LargeAttachment", "FitConnect.LargeAttachment\FitConnect.LargeAttachment.csproj", "{8DBA0593-12DC-4791-AA66-BAD9B6573CC7}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -68,6 +70,10 @@ Global
{DDEFDA59-FD75-4F25-9BA2-8DAE22B4A0B0}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DDEFDA59-FD75-4F25-9BA2-8DAE22B4A0B0}.Release|Any CPU.ActiveCfg = Release|Any CPU
{DDEFDA59-FD75-4F25-9BA2-8DAE22B4A0B0}.Release|Any CPU.Build.0 = Release|Any CPU
+ {8DBA0593-12DC-4791-AA66-BAD9B6573CC7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {8DBA0593-12DC-4791-AA66-BAD9B6573CC7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {8DBA0593-12DC-4791-AA66-BAD9B6573CC7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {8DBA0593-12DC-4791-AA66-BAD9B6573CC7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{27115A99-2AE8-42BC-9495-BE2DCEDDF1E8} = {180029B5-8DD3-4594-B34E-6C07AF1C52C5}
diff --git a/FitConnect/Models/Attachment.cs b/FitConnect/Models/Attachment.cs
index e956b72a..832fd6ce 100644
--- a/FitConnect/Models/Attachment.cs
+++ b/FitConnect/Models/Attachment.cs
@@ -49,9 +49,12 @@ public class Attachment : AttachmentMetadata {
MimeType = MediaTypeNames.Application.Octet;
}
+ protected Attachment() {
+ }
+
public Guid Id { get; } = Guid.NewGuid();
- private byte[]? Content { get; init; }
+ protected byte[]? Content { get; set; }
public string? AttachmentAuthentication { get; internal set; }
diff --git a/FitConnect/Sender.cs b/FitConnect/Sender.cs
index 11420b48..6520687a 100644
--- a/FitConnect/Sender.cs
+++ b/FitConnect/Sender.cs
@@ -155,7 +155,7 @@ public class Sender : FitConnectClient, ISender {
Logger?.LogInformation("Metadata validation check, done");
Logger?.LogInformation("Sending submission");
var encryptedMeta = Encryption.Encrypt(metadata);
- Logger?.LogTrace("Encrypted metadata: {EncryptedMeta}", encryptedMeta);
+ // Logger?.LogTrace("Encrypted metadata: {EncryptedMeta}", encryptedMeta);
submission.EncryptedMetadata = encryptedMeta;
}
}
@@ -246,40 +246,6 @@ public class Sender : FitConnectClient, ISender {
return base.GetEventLogAsync(submission, true);
}
- private async Task<Submission> Submit(Submission submission) {
- if (submission == null) {
- Logger?.LogCritical("Submission is null on submit");
- throw new InvalidOperationException("Submission is not ready");
- }
-
- if (submission.EncryptedMetadata == null) {
- var metadata = CreateMetadata(submission);
- submission.GeneratedMetadata = metadata;
- Logger?.LogTrace("Metadata: {Metadata}", metadata);
- var valid = await JsonHelper.VerifyMetadata(metadata, null, Logger);
- if (!valid) {
- Logger?.LogError("Sending submission aborted due to validation errors");
- throw new InvalidOperationException("Submission is not ready");
- }
-
- Logger?.LogInformation("Metadata validation check, done");
- Logger?.LogInformation("Sending submission");
- var encryptedMeta = Encryption.Encrypt(metadata);
- Logger?.LogTrace("Encrypted metadata: {EncryptedMeta}", encryptedMeta);
- submission.EncryptedMetadata = encryptedMeta;
- }
-
- if (submission.EncryptedData == null)
- if (submission.Data != null)
- submission.EncryptedData = Encryption.Encrypt(submission.Data);
-
- var result = await SubmissionService
- .SubmitSubmission(submission.Id!, (SubmitSubmissionDto)submission);
-
- Logger?.LogInformation("Submission sent");
- return submission;
- }
-
/// <summary>
/// Creates a new <see cref="SendableSubmission" /> on the FitConnect server.
/// </summary>
diff --git a/FitConnect/Services/Models/v1/Api/MetadataDto.cs b/FitConnect/Services/Models/v1/Api/MetadataDto.cs
index 699f9ed5..a9c2a18b 100644
--- a/FitConnect/Services/Models/v1/Api/MetadataDto.cs
+++ b/FitConnect/Services/Models/v1/Api/MetadataDto.cs
@@ -117,7 +117,7 @@ namespace FitConnect.Models.Api.Metadata
/// Optionale Beschreibung der Anlage
/// </summary>
[JsonProperty("description", NullValueHandling = NullValueHandling.Ignore)]
- public string Description { get; internal set; }
+ public string Description { get; protected internal set; }
/// <summary>
/// Ursprünglicher Dateiname bei Erzeugung oder Upload
@@ -125,7 +125,7 @@ namespace FitConnect.Models.Api.Metadata
[JsonProperty("filename", NullValueHandling = NullValueHandling.Ignore)]
public string Filename {
get => _filename;
- internal set => _filename = Path.GetFileName(value);
+ protected internal set => _filename = Path.GetFileName(value);
}
/// <summary>
@@ -140,7 +140,7 @@ namespace FitConnect.Models.Api.Metadata
/// Internet Media Type gemäß RFC 2045, z. B. application/pdf.
/// </summary>
[JsonProperty("mimeType")]
- public string MimeType { get; internal set; }
+ public string MimeType { get; protected internal set; }
/// <summary>
/// Zweck/Art der Anlage
diff --git a/FitConnect/Services/RestCallService.cs b/FitConnect/Services/RestCallService.cs
index af4f2819..fd3787ab 100644
--- a/FitConnect/Services/RestCallService.cs
+++ b/FitConnect/Services/RestCallService.cs
@@ -83,7 +83,7 @@ internal class RestCallService : IRestCallService {
if (body != null) {
request.Content = new StringContent(body, Encoding.UTF8, contentType);
- _logger?.LogTrace("Body: {Body}", body);
+ // _logger?.LogTrace("Body: {Body}", body);
}
var response = await client.SendAsync(request);
@@ -96,7 +96,7 @@ internal class RestCallService : IRestCallService {
if (response.IsSuccessStatusCode) {
var content = await response.Content.ReadAsStringAsync();
- _logger?.LogTrace("Response: {Content}", content);
+ // _logger?.LogTrace("Response: {Content}", content);
return content;
}
diff --git a/Tests/BasicUnitTest/BasicUnitTest.csproj b/Tests/BasicUnitTest/BasicUnitTest.csproj
index 0cc5b3ee..e81004dd 100644
--- a/Tests/BasicUnitTest/BasicUnitTest.csproj
+++ b/Tests/BasicUnitTest/BasicUnitTest.csproj
@@ -26,6 +26,7 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
+ <PackageReference Include="SpecFlow.NUnit" Version="3.9.74" />
</ItemGroup>
<ItemGroup>
--
GitLab