diff --git a/src/api/BoreholeFileCloudService.cs b/src/api/BoreholeFileCloudService.cs index d7fea3e22..95330bc4b 100644 --- a/src/api/BoreholeFileCloudService.cs +++ b/src/api/BoreholeFileCloudService.cs @@ -31,9 +31,11 @@ public BoreholeFileCloudService(BdmsContext context, IConfiguration configuratio /// /// Uploads a file to the cloud storage and links it to the borehole. /// - /// The file to upload and link to the . - /// The to link the uploaded to. - public async Task UploadFileAndLinkToBorehole(IFormFile formFile, int boreholeId) + /// The file stream for the file to upload and link to the . + /// The name of the file to upload. + /// The content type of the file. + /// The to link the uploaded file to. + public async Task UploadFileAndLinkToBoreholeAsync(Stream fileStream, string fileName, string contentType, int boreholeId) { // Use transaction to ensure data is only stored to db if the file upload was sucessful. Only create a transaction if there is not already one from the calling method. using var transaction = context.Database.CurrentTransaction == null ? await context.Database.BeginTransactionAsync().ConfigureAwait(false) : null; @@ -49,10 +51,10 @@ public async Task UploadFileAndLinkToBorehole(IFormFile formFile, if (user == null || subjectId == null) throw new InvalidOperationException($"No user with subject_id <{subjectId}> found."); // Register the new file in the boreholes database. - var fileExtension = Path.GetExtension(formFile.FileName); + var fileExtension = Path.GetExtension(fileName); var fileNameGuid = $"{Guid.NewGuid()}{fileExtension}"; - var file = new Models.File { Name = formFile.FileName, NameUuid = fileNameGuid, Type = formFile.ContentType }; + var file = new Models.File { Name = fileName, NameUuid = fileNameGuid, Type = contentType }; await context.Files.AddAsync(file).ConfigureAwait(false); await context.UpdateChangeInformationAndSaveChangesAsync(httpContextAccessor.HttpContext!).ConfigureAwait(false); @@ -60,11 +62,11 @@ public async Task UploadFileAndLinkToBorehole(IFormFile formFile, var fileId = file.Id; // Upload the file to the cloud storage. - await UploadObject(formFile, fileNameGuid).ConfigureAwait(false); + await UploadObject(fileStream, fileNameGuid, contentType).ConfigureAwait(false); // If file is already linked to the borehole, throw an exception. if (await context.BoreholeFiles.AnyAsync(bf => bf.BoreholeId == boreholeId && bf.FileId == fileId).ConfigureAwait(false)) - throw new InvalidOperationException($"File <{formFile.FileName}> is already attached to borehole with Id <{boreholeId}>."); + throw new InvalidOperationException($"File <{fileName}> is already attached to borehole with Id <{boreholeId}>."); // Link file to the borehole. var boreholeFile = new BoreholeFile { FileId = fileId, BoreholeId = boreholeId, UserId = user.Id, Attached = DateTime.UtcNow }; @@ -77,7 +79,7 @@ public async Task UploadFileAndLinkToBorehole(IFormFile formFile, } catch (Exception ex) { - logger.LogError(ex, "Error attaching file <{FileName}> to borehole with Id <{BoreholeId}>.", formFile.FileName, boreholeId); + logger.LogError(ex, "Error attaching file <{FileName}> to borehole with Id <{BoreholeId}>.", fileName, boreholeId); throw; } } @@ -85,14 +87,20 @@ public async Task UploadFileAndLinkToBorehole(IFormFile formFile, /// /// Uploads a file to the cloud storage. /// - /// The file to upload. + /// The file stream to upload. /// The name of the file in the storage. - internal async Task UploadObject(IFormFile file, string objectName) + /// The content type of the file. + internal async Task UploadObject(Stream fileStream, string objectName, string contentType) { try { - // Upload file - var putObjectRequest = new PutObjectRequest { BucketName = bucketName, Key = objectName, InputStream = file.OpenReadStream(), ContentType = file.ContentType }; + var putObjectRequest = new PutObjectRequest + { + BucketName = bucketName, + Key = objectName, + InputStream = fileStream, + ContentType = contentType, + }; await s3Client.PutObjectAsync(putObjectRequest).ConfigureAwait(false); } catch (AmazonS3Exception ex) diff --git a/src/api/Controllers/BoreholeFileController.cs b/src/api/Controllers/BoreholeFileController.cs index bfe711ccc..0f372f64c 100644 --- a/src/api/Controllers/BoreholeFileController.cs +++ b/src/api/Controllers/BoreholeFileController.cs @@ -45,7 +45,7 @@ public async Task Upload(IFormFile file, [Range(1, int.MaxValue)] try { - var boreholeFile = await boreholeFileCloudService.UploadFileAndLinkToBorehole(file, boreholeId).ConfigureAwait(false); + var boreholeFile = await boreholeFileCloudService.UploadFileAndLinkToBoreholeAsync(file.OpenReadStream(), file.FileName, file.ContentType, boreholeId).ConfigureAwait(false); return Ok(boreholeFile); } catch (InvalidOperationException ex) diff --git a/src/api/Controllers/ImportController.cs b/src/api/Controllers/ImportController.cs index f6f17b327..27a68d5bf 100644 --- a/src/api/Controllers/ImportController.cs +++ b/src/api/Controllers/ImportController.cs @@ -307,47 +307,35 @@ private async Task UploadAttachmentsAsync(ZipArchive zipArchive, List e.FullName == fileName); + if (attachment == null) + { + AddValidationErrorToModelState(borehole.index, $"Attachment with the name <{fileName}> is referenced in JSON file but was not not found in ZIP archive.", ValidationErrorType.Attachment); + continue; + } using var memoryStream = new MemoryStream(); - FormFile formFile = await CreateFormFileFromAttachmentAsync(fileToProcess, attachment, memoryStream).ConfigureAwait(false); + await attachment.Open().CopyToAsync(memoryStream).ConfigureAwait(false); + memoryStream.Position = 0; // Remove original file information from borehole object borehole.value.BoreholeFiles.Remove(fileToProcess); - - await UploadFormFileAsync(borehole.value, borehole.index, fileName, formFile).ConfigureAwait(false); + await UploadFormFileAsync(memoryStream, fileToProcess.File.Name, GetContentType(attachment.Name), borehole).ConfigureAwait(false); } } } } - private async Task UploadFormFileAsync(BoreholeImport borehole, int boreholeIndex, string fileName, FormFile formFile) + private async Task UploadFormFileAsync(Stream fileStream, string fileName, string contentType, (BoreholeImport Value, int Index) borehole) { try { - await boreholeFileCloudService.UploadFileAndLinkToBorehole(formFile, borehole.Id).ConfigureAwait(false); + await boreholeFileCloudService.UploadFileAndLinkToBoreholeAsync(fileStream, fileName, contentType, borehole.Value.Id).ConfigureAwait(false); } catch (Exception ex) { - logger.LogError(ex, "An error occurred while uploading the file: {FileName}", formFile.FileName); - AddValidationErrorToModelState(boreholeIndex, string.Format(CultureInfo.InvariantCulture, $"An error occurred while uploading the file: <{fileName}>", "upload"), ValidationErrorType.Attachment); - } - } - - private static async Task CreateFormFileFromAttachmentAsync(BoreholeFile? boreholeFile, ZipArchiveEntry? attachment, MemoryStream memoryStream) - { - using (var entryStream = attachment.Open()) - { - await entryStream.CopyToAsync(memoryStream).ConfigureAwait(false); + logger.LogError(ex, "An error occurred while uploading the file: {FileName}", fileName); + AddValidationErrorToModelState(borehole.Index, string.Format(CultureInfo.InvariantCulture, $"An error occurred while uploading the file: <{fileName}>", "upload"), ValidationErrorType.Attachment); } - - memoryStream.Position = 0; - - var formFile = new FormFile(memoryStream, 0, memoryStream.Length, boreholeFile.File.Name, boreholeFile.File.Name) - { - Headers = new HeaderDictionary(), - ContentType = GetContentType(attachment.Name), - }; - return formFile; } private static string GetContentType(string fileName) diff --git a/tests/api/BoreholeFileCloudServiceTest.cs b/tests/api/BoreholeFileCloudServiceTest.cs index 2751e6269..5f7d67104 100644 --- a/tests/api/BoreholeFileCloudServiceTest.cs +++ b/tests/api/BoreholeFileCloudServiceTest.cs @@ -63,7 +63,7 @@ public async Task UploadFileAndLinkToBoreholeShouldStoreFileInCloudStorageAndLin var firstPdfFormFile = GetFormFileByContent(content, fileName); // Upload file - await boreholeFileUploadService.UploadFileAndLinkToBorehole(firstPdfFormFile, minBoreholeId); + await boreholeFileUploadService.UploadFileAndLinkToBoreholeAsync(firstPdfFormFile.OpenReadStream(), firstPdfFormFile.FileName, firstPdfFormFile.ContentType, minBoreholeId).ConfigureAwait(false); // Get borehole with file linked from db var borehole = GetBoreholesWithIncludes(context.Boreholes).Single(b => b.Id == minBoreholeId); @@ -85,7 +85,7 @@ public async Task UploadObjectShouldStoreFileInCloudStorage() var pdfFormFile = GetFormFileByContent(Guid.NewGuid().ToString(), fileName); // Upload file - await boreholeFileUploadService.UploadObject(pdfFormFile, pdfFormFile.FileName); + await boreholeFileUploadService.UploadObject(pdfFormFile.OpenReadStream(), pdfFormFile.FileName, pdfFormFile.ContentType); // Get all objects in the bucket with provided name var listObjectsRequest = new ListObjectsV2Request { BucketName = bucketName, MaxKeys = 1000, Prefix = fileName }; @@ -105,7 +105,7 @@ public async Task UploadObjectSameFileTwiceShouldReplaceFileInCloudStorage() var pdfFormFile = GetFormFileByContent(content, "file_1.pdf"); // First Upload file - await boreholeFileUploadService.UploadObject(pdfFormFile, pdfFormFile.FileName); + await boreholeFileUploadService.UploadObject(pdfFormFile.OpenReadStream(), pdfFormFile.FileName, pdfFormFile.ContentType); // Get all files with same key after upload var listObjectsRequest = new ListObjectsV2Request { BucketName = bucketName, MaxKeys = 1000, Prefix = pdfFormFile.FileName }; @@ -119,7 +119,7 @@ public async Task UploadObjectSameFileTwiceShouldReplaceFileInCloudStorage() var uploadDate = files.First().LastModified; // Second Upload file - await boreholeFileUploadService.UploadObject(pdfFormFile, pdfFormFile.FileName); + await boreholeFileUploadService.UploadObject(pdfFormFile.OpenReadStream(), pdfFormFile.FileName, pdfFormFile.ContentType); // Get all objects in the bucket with provided name listObjectResponse = await s3Client.ListObjectsV2Async(listObjectsRequest).ConfigureAwait(false); @@ -146,7 +146,7 @@ public async Task GetObjectShouldReturnFileBytes() var pdfFormFile = GetFormFileByContent(content, "file_1.pdf"); // Upload file - await boreholeFileUploadService.UploadObject(pdfFormFile, pdfFormFile.FileName); + await boreholeFileUploadService.UploadObject(pdfFormFile.OpenReadStream(), pdfFormFile.FileName, pdfFormFile.ContentType); // Download file var result = await boreholeFileUploadService.GetObject(pdfFormFile.FileName); @@ -161,7 +161,7 @@ public async Task DeleteObjectShouldDeleteObjectFromStorage() var pdfFormFile = GetFormFileByContent(content, "file_1.pdf"); // Upload file - await boreholeFileUploadService.UploadObject(pdfFormFile, pdfFormFile.FileName); + await boreholeFileUploadService.UploadObject(pdfFormFile.OpenReadStream(), pdfFormFile.FileName, pdfFormFile.ContentType); // Ensure file exists await boreholeFileUploadService.GetObject(pdfFormFile.FileName); diff --git a/tests/api/Controllers/BoreholeFileControllerTest.cs b/tests/api/Controllers/BoreholeFileControllerTest.cs index 75f432c63..07f3adcaf 100644 --- a/tests/api/Controllers/BoreholeFileControllerTest.cs +++ b/tests/api/Controllers/BoreholeFileControllerTest.cs @@ -356,11 +356,11 @@ public async Task GetDataExtractionInfo() var fileUuid = file.File.NameUuid.Replace(".pdf", ""); var image1 = GetFormFileByExistingFile("labeling_attachment-1.png"); - await boreholeFileCloudService.UploadObject(image1, $"dataextraction/{fileUuid}-1.png"); + await boreholeFileCloudService.UploadObject(image1.OpenReadStream(), $"dataextraction/{fileUuid}-1.png", image1.ContentType); var image2 = GetFormFileByExistingFile("labeling_attachment-2.png"); - await boreholeFileCloudService.UploadObject(image2, $"dataextraction/{fileUuid}-2.png"); + await boreholeFileCloudService.UploadObject(image2.OpenReadStream(), $"dataextraction/{fileUuid}-2.png", image2.ContentType); var image3 = GetFormFileByExistingFile("labeling_attachment-3.png"); - await boreholeFileCloudService.UploadObject(image3, $"dataextraction/{fileUuid}-3.png"); + await boreholeFileCloudService.UploadObject(image3.OpenReadStream(), $"dataextraction/{fileUuid}-3.png", image3.ContentType); // Test var result = await controller.GetDataExtractionFileInfo(file.FileId, 1); @@ -436,7 +436,7 @@ public async Task GetDataExtractionImage() var fileUuid = file.File.NameUuid.Replace(".pdf", ""); var image1 = GetFormFileByExistingFile("labeling_attachment-1.png"); - await boreholeFileCloudService.UploadObject(image1, $"dataextraction/{fileUuid}-1.png"); + await boreholeFileCloudService.UploadObject(image1.OpenReadStream(), $"dataextraction/{fileUuid}-1.png", image1.ContentType); // Test var response = await controller.GetDataExtractionImage($"{fileUuid}-1.png"); diff --git a/tests/api/Controllers/ExportControllerTest.cs b/tests/api/Controllers/ExportControllerTest.cs index eae326272..221007c30 100644 --- a/tests/api/Controllers/ExportControllerTest.cs +++ b/tests/api/Controllers/ExportControllerTest.cs @@ -177,8 +177,7 @@ public async Task ExportJsonWithAttachments() var fileName = $"{Guid.NewGuid()}.pdf"; var content = Guid.NewGuid().ToString(); var fileBytes = Encoding.UTF8.GetBytes(content); - var formFile = new FormFile(new MemoryStream(fileBytes), 0, fileBytes.Length, null, fileName) { Headers = new HeaderDictionary(), ContentType = "application/pdf" }; - var boreholeFile = await boreholeFileCloudService.UploadFileAndLinkToBorehole(formFile, newBorehole.Id).ConfigureAwait(false); + var boreholeFile = await boreholeFileCloudService.UploadFileAndLinkToBoreholeAsync(new MemoryStream(fileBytes), fileName, "application/pdf", newBorehole.Id).ConfigureAwait(false); context.BoreholeFiles.Add(boreholeFile); var result = await controller.ExportJsonWithAttachmentsAsync([newBorehole.Id]).ConfigureAwait(false);