diff --git a/src/Altinn.App.Core/Internal/Process/ProcessTasks/Common/ProcessTaskFinalizer.cs b/src/Altinn.App.Core/Internal/Process/ProcessTasks/Common/ProcessTaskFinalizer.cs index 0ce5a8cf7..635e6088f 100644 --- a/src/Altinn.App.Core/Internal/Process/ProcessTasks/Common/ProcessTaskFinalizer.cs +++ b/src/Altinn.App.Core/Internal/Process/ProcessTasks/Common/ProcessTaskFinalizer.cs @@ -136,6 +136,9 @@ private async Task RemoveFieldsOnTaskComplete( // Remove hidden data before validation, ignore hidden rows. if (_appSettings.Value?.RemoveHiddenData == true) { + // Backend removal of data is deprecated in favor of + // implementing frontend removal of hidden data, so + //this is not updated to remove from multiple data models at once. LayoutEvaluatorState evaluationState = await _layoutEvaluatorStateInitializer.Init( instance, dataAccessor, @@ -184,12 +187,13 @@ await _dataClient.InsertFormData( else { // Remove the shadow fields from the data - // TODO: This does not work!!! + data = JsonSerializer.Deserialize(serializedData, data.GetType()) ?? throw new JsonException( "Could not deserialize back datamodel after removing shadow fields. Data was \"null\"" ); + (dataAccessor as CachedInstanceDataAccessor)?.Set(dataElement, data); isModified = true; // TODO: Detect if modifications were made } } diff --git a/test/Altinn.App.Api.Tests/Controllers/ProcessControllerTests.cs b/test/Altinn.App.Api.Tests/Controllers/ProcessControllerTests.cs index 870092dc5..567d1b9d6 100644 --- a/test/Altinn.App.Api.Tests/Controllers/ProcessControllerTests.cs +++ b/test/Altinn.App.Api.Tests/Controllers/ProcessControllerTests.cs @@ -2,14 +2,20 @@ using System.Text; using System.Text.Encodings.Web; using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; using Altinn.App.Api.Models; using Altinn.App.Api.Tests.Data; +using Altinn.App.Api.Tests.Data.apps.tdd.contributer_restriction.models; using Altinn.App.Core.Features; +using Altinn.App.Core.Internal.Instances; using Altinn.App.Core.Internal.Pdf; using Altinn.App.Core.Models.Validation; using Altinn.Platform.Storage.Interface.Models; +using App.IntegrationTests.Mocks.Services; using FluentAssertions; +using Json.Patch; +using Json.Pointer; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.Extensions.DependencyInjection; using Moq; @@ -23,9 +29,9 @@ public class ProcessControllerTests : ApiTestBase, IClassFixture factory, ITestOutpu services.AddSingleton(_dataProcessorMock.Object); services.AddSingleton(_formDataValidatorMock.Object); }; - TestData.DeleteInstanceAndData(Org, App, InstanceOwnerPartyId, InstanceGuid); - TestData.PrepareInstance(Org, App, InstanceOwnerPartyId, InstanceGuid); + TestData.DeleteInstanceAndData(Org, App, InstanceOwnerPartyId, _instanceGuid); + TestData.PrepareInstance(Org, App, InstanceOwnerPartyId, _instanceGuid); } [Fact] @@ -146,7 +152,7 @@ public async Task RunProcessNextWithLang_VerifyPdfCallWithLanguage() using var client = GetRootedClient(Org, App, 1337, InstanceOwnerPartyId); // both "?lang" and "?language" should work var nextResponse = await client.PutAsync( - $"{Org}/{App}/instances/{InstanceId}/process/next?lang={language}", + $"{Org}/{App}/instances/{_instanceId}/process/next?lang={language}", null ); var nextResponseContent = await nextResponse.Content.ReadAsStringAsync(); @@ -179,7 +185,7 @@ public async Task RunProcessNextWithLanguage_VerifyPdfCall() using var client = GetRootedClient(Org, App, 1337, InstanceOwnerPartyId); // both "?lang" and "?language" should work var nextResponse = await client.PutAsync( - $"{Org}/{App}/instances/{InstanceId}/process/next?language={language}", + $"{Org}/{App}/instances/{_instanceId}/process/next?language={language}", null ); var nextResponseContent = await nextResponse.Content.ReadAsStringAsync(); @@ -191,7 +197,7 @@ public async Task RunProcessNextWithLanguage_VerifyPdfCall() public async Task RunProcessNext_PdfFails_DataIsUnlocked() { bool sendAsyncCalled = false; - var dataElementPath = TestData.GetDataElementPath(Org, App, InstanceOwnerPartyId, InstanceGuid, DataGuid); + var dataElementPath = TestData.GetDataElementPath(Org, App, InstanceOwnerPartyId, _instanceGuid, _dataGuid); SendAsync = async message => { @@ -213,7 +219,7 @@ public async Task RunProcessNext_PdfFails_DataIsUnlocked() return new HttpResponseMessage(HttpStatusCode.TooManyRequests); }; using var client = GetRootedClient(Org, App, 1337, InstanceOwnerPartyId); - var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{InstanceId}/process/next", null); + var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{_instanceId}/process/next", null); var nextResponseContent = await nextResponse.Content.ReadAsStringAsync(); _outputHelper.WriteLine(nextResponseContent); nextResponse.Should().HaveStatusCode(HttpStatusCode.InternalServerError); @@ -256,7 +262,7 @@ public async Task RunProcessNext_FailingValidator_ReturnsValidationErrors() services.AddSingleton(dataValidator.Object); }; using var client = GetRootedClient(Org, App, 1337, InstanceOwnerPartyId); - var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{InstanceId}/process/next", null); + var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{_instanceId}/process/next", null); var nextResponseContent = await nextResponse.Content.ReadAsStringAsync(); _outputHelper.WriteLine(nextResponseContent); nextResponse.Should().HaveStatusCode(HttpStatusCode.Conflict); @@ -270,11 +276,185 @@ public async Task RunProcessNext_FailingValidator_ReturnsValidationErrors() ); // Verify that the instance is not updated - var instance = await TestData.GetInstance(Org, App, InstanceOwnerPartyId, InstanceGuid); + var instance = await TestData.GetInstance(Org, App, InstanceOwnerPartyId, _instanceGuid); instance.Process.CurrentTask.Should().NotBeNull(); instance.Process.CurrentTask!.ElementId.Should().Be("Task_1"); } + [Fact] + public async Task RunProcessNext_DataFromHiddenComponents_GetsRemoved() + { + // Override config to remove hidden data + OverrideAppSetting("AppSettings:RemoveHiddenData", "true"); + + // Mock pdf generation so that the test does not fail due to pof service not running. + var pdfMock = new Mock(MockBehavior.Strict); + using var pdfReturnStream = new MemoryStream(); + pdfMock.Setup(p => p.GeneratePdf(It.IsAny(), It.IsAny())).ReturnsAsync(pdfReturnStream); + OverrideServicesForThisTest = (services) => + { + services.AddSingleton(pdfMock.Object); + }; + // setup data processor + _dataProcessorMock + .Setup(dp => + dp.ProcessDataWrite( + It.IsAny(), + _dataGuid, + It.IsAny(), + It.IsAny(), + It.IsAny() + ) + ) + .Returns(Task.CompletedTask) + .Verifiable(Times.Once); + + // create client for tests + using var client = GetRootedClient(Org, App, 1337, InstanceOwnerPartyId); + var dataPath = TestData.GetDataBlobPath(Org, App, InstanceOwnerPartyId, _instanceGuid, _dataGuid); + + // Update hidden data value + var serializedPatch = JsonSerializer.Serialize( + new DataPatchRequest() + { + Patch = new JsonPatch( + PatchOperation.Add( + JsonPointer.Create("melding", "hidden"), + JsonNode.Parse("\"value that is hidden\"") + ) + ), + IgnoredValidators = [] + }, + _jsonSerializerOptions + ); + _outputHelper.WriteLine(serializedPatch); + using var updateDataElementContent = new StringContent(serializedPatch, Encoding.UTF8, "application/json"); + using var response = await client.PatchAsync( + $"{Org}/{App}/instances/{InstanceOwnerPartyId}/{_instanceGuid}/data/{_dataGuid}", + updateDataElementContent + ); + response.Should().HaveStatusCode(HttpStatusCode.OK); + + // Verify that hidden is stored + var dataString = await File.ReadAllTextAsync(dataPath); + _outputHelper.WriteLine("Data before process next:"); + _outputHelper.WriteLine(dataString); + dataString.Should().Contain("value that is hidden"); + + // Run process next + var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{_instanceId}/process/next", null); + var nextResponseContent = await nextResponse.Content.ReadAsStringAsync(); + _outputHelper.WriteLine(nextResponseContent); + nextResponse.Should().HaveStatusCode(HttpStatusCode.OK); + + // Verify that the instance is updated to the ended state + dataString = await File.ReadAllTextAsync(dataPath); + _outputHelper.WriteLine("Data after process next:"); + _outputHelper.WriteLine(dataString); + dataString.Should().NotContain("value that is hidden"); + + _dataProcessorMock.Verify(); + } + + [Theory] + [InlineData(null)] + [InlineData("copyDataType")] + public async Task RunProcessNext_ShadowFields_GetsRemoved(string? saveToDataType) + { + // Mock pdf generation so that the test does not fail due to pof service not running. + var pdfMock = new Mock(MockBehavior.Strict); + using var pdfReturnStream = new MemoryStream(); + pdfMock.Setup(p => p.GeneratePdf(It.IsAny(), It.IsAny())).ReturnsAsync(pdfReturnStream); + OverrideServicesForThisTest = (services) => + { + services.AddSingleton(pdfMock.Object); + services.AddSingleton( + new AppMetadataMutationHook(appMetadata => + { + var defaultDataType = appMetadata.DataTypes.Single(dt => dt.Id == "default"); + defaultDataType.AppLogic.ShadowFields = new() { Prefix = "SF_", SaveToDataType = saveToDataType }; + + if (saveToDataType is not null) + appMetadata.DataTypes.Add( + new DataType() + { + Id = saveToDataType, + TaskId = "Task_1", + AppLogic = new() { ClassRef = defaultDataType.AppLogic.ClassRef } + } + ); + }) + ); + }; + // setup data processor + _dataProcessorMock + .Setup(dp => + dp.ProcessDataWrite( + It.IsAny(), + _dataGuid, + It.IsAny(), + It.IsAny(), + It.IsAny() + ) + ) + .Returns(Task.CompletedTask) + .Verifiable(Times.Once); + + // create client for tests + using var client = GetRootedClient(Org, App, 1337, InstanceOwnerPartyId); + + // Update hidden data value + var serializedPatch = JsonSerializer.Serialize( + new DataPatchRequest() + { + Patch = new JsonPatch( + PatchOperation.Add( + JsonPointer.Create("melding", "SF_test"), + JsonNode.Parse("\"value that is in shadow field\"") + ) + ), + IgnoredValidators = [] + }, + _jsonSerializerOptions + ); + _outputHelper.WriteLine(serializedPatch); + using var updateDataElementContent = new StringContent(serializedPatch, Encoding.UTF8, "application/json"); + using var response = await client.PatchAsync( + $"{Org}/{App}/instances/{InstanceOwnerPartyId}/{_instanceGuid}/data/{_dataGuid}", + updateDataElementContent + ); + response.Should().HaveStatusCode(HttpStatusCode.OK); + + // Verify that hidden is stored + var dataPath = TestData.GetDataBlobPath(Org, App, InstanceOwnerPartyId, _instanceGuid, _dataGuid); + var dataString = await File.ReadAllTextAsync(dataPath); + _outputHelper.WriteLine("Data before process next:"); + _outputHelper.WriteLine(dataString); + dataString.Should().Contain("value that is in shadow field"); + + // Run process next + using var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{_instanceId}/process/next", null); + var nextResponseContent = await nextResponse.Content.ReadAsStringAsync(); + _outputHelper.WriteLine(nextResponseContent); + nextResponse.Should().HaveStatusCode(HttpStatusCode.OK); + + // Get data path if the data element with shadow fields removed is saved to another data type + if (saveToDataType is not null) + { + var instanceClient = Services.GetRequiredService(); + var instance = await instanceClient.GetInstance(App, Org, InstanceOwnerPartyId, _instanceGuid); + var copyDataGuid = Guid.Parse(instance.Data.Single(de => de.DataType == saveToDataType).Id); + dataPath = TestData.GetDataBlobPath(Org, App, InstanceOwnerPartyId, _instanceGuid, copyDataGuid); + } + // Verify that the instance is updated to the ended state + dataString = await File.ReadAllTextAsync(dataPath); + _outputHelper.WriteLine("Data after process next:"); + _outputHelper.WriteLine(dataString); + dataString.Should().NotContain("value that is in shadow field"); + + _dataProcessorMock.Verify(); + } + [Fact] public async Task RunProcessNext_NonErrorValidations_ReturnsOk() { @@ -328,7 +508,7 @@ public async Task RunProcessNext_NonErrorValidations_ReturnsOk() services.AddSingleton(pdfMock.Object); }; using var client = GetRootedClient(Org, App, 1337, InstanceOwnerPartyId); - var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{InstanceId}/process/next", null); + var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{_instanceId}/process/next", null); var nextResponseContent = await nextResponse.Content.ReadAsStringAsync(); _outputHelper.WriteLine(nextResponseContent); nextResponse.Should().HaveStatusCode(HttpStatusCode.OK); @@ -336,7 +516,7 @@ public async Task RunProcessNext_NonErrorValidations_ReturnsOk() document.RootElement.EnumerateObject().Should().NotContain(p => p.Name == "validationIssues"); // Verify that the instance is updated to the ended state - var instance = await TestData.GetInstance(Org, App, InstanceOwnerPartyId, InstanceGuid); + var instance = await TestData.GetInstance(Org, App, InstanceOwnerPartyId, _instanceGuid); instance.Process.CurrentTask.Should().BeNull(); instance.Process.EndEvent.Should().Be("EndEvent_1"); } @@ -352,13 +532,13 @@ public async Task RunCompleteTask_GoesToEndEvent() services.AddSingleton(pdfMock.Object); }; using var client = GetRootedClient(Org, App, 1337, InstanceOwnerPartyId); - var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{InstanceId}/process/completeProcess", null); + var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{_instanceId}/process/completeProcess", null); var nextResponseContent = await nextResponse.Content.ReadAsStringAsync(); _outputHelper.WriteLine(nextResponseContent); nextResponse.Should().HaveStatusCode(HttpStatusCode.OK); // Verify that the instance is updated to the ended state - var instance = await TestData.GetInstance(Org, App, InstanceOwnerPartyId, InstanceGuid); + var instance = await TestData.GetInstance(Org, App, InstanceOwnerPartyId, _instanceGuid); instance.Process.CurrentTask.Should().BeNull(); instance.Process.EndEvent.Should().Be("EndEvent_1"); } @@ -379,7 +559,7 @@ public async Task RunNextWithAction_WhenActionIsNotAuthorized_ReturnsUnauthorize Encoding.UTF8, "application/json" ); - var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{InstanceId}/process/next", content); + var nextResponse = await client.PutAsync($"{Org}/{App}/instances/{_instanceId}/process/next", content); var nextResponseContent = await nextResponse.Content.ReadAsStringAsync(); _outputHelper.WriteLine(nextResponseContent); nextResponse.Should().HaveStatusCode(HttpStatusCode.Forbidden); diff --git a/test/Altinn.App.Api.Tests/CustomWebApplicationFactory.cs b/test/Altinn.App.Api.Tests/CustomWebApplicationFactory.cs index b1435f869..c2d64af46 100644 --- a/test/Altinn.App.Api.Tests/CustomWebApplicationFactory.cs +++ b/test/Altinn.App.Api.Tests/CustomWebApplicationFactory.cs @@ -58,7 +58,10 @@ public HttpClient GetRootedClient(string org, string app, bool includeTraceConte var factory = _factory.WithWebHostBuilder(builder => { - var configuration = new ConfigurationBuilder().AddJsonFile(appSettingsPath).Build(); + var configuration = new ConfigurationBuilder() + .AddJsonFile(appSettingsPath) + .AddInMemoryCollection(_configOverrides) + .Build(); configuration.GetSection("AppSettings:AppBasePath").Value = appRootPath; IConfigurationSection appSettingSection = configuration.GetSection("AppSettings"); @@ -79,6 +82,16 @@ public HttpClient GetRootedClient(string org, string app, bool includeTraceConte return client; } + /// + /// Overrides the app settings for the test application. + /// + public void OverrideAppSetting(string key, string? value) + { + _configOverrides[key] = value; + } + + private readonly Dictionary _configOverrides = new(); + private sealed class DiagnosticHandler : DelegatingHandler { protected override Task SendAsync( diff --git a/test/Altinn.App.Api.Tests/Data/apps/tdd/contributer-restriction/models/Skjema.cs b/test/Altinn.App.Api.Tests/Data/apps/tdd/contributer-restriction/models/Skjema.cs index 5d299395b..9a10f272b 100644 --- a/test/Altinn.App.Api.Tests/Data/apps/tdd/contributer-restriction/models/Skjema.cs +++ b/test/Altinn.App.Api.Tests/Data/apps/tdd/contributer-restriction/models/Skjema.cs @@ -1,3 +1,4 @@ +#pragma warning disable IDE1006 // Naming Styles does not matter in model classes using System.ComponentModel.DataAnnotations; using System.Text.Json.Serialization; using System.Xml.Serialization; @@ -55,6 +56,16 @@ public bool ShouldSerializeTagWithAttribute() { return TagWithAttribute?.value != null; } + + [XmlElement("hidden", Order = 8)] + [JsonProperty("hidden")] + [JsonPropertyName("hidden")] + public string? Hidden { get; set; } + + [XmlElement("SF_test", Order = 9)] + [JsonProperty("SF_test")] + [JsonPropertyName("SF_test")] + public string? SF_test { get; set; } } public class TagWithAttribute @@ -129,3 +140,5 @@ public bool AltinnRowIdSpecified() [JsonPropertyName("values")] public List? Values { get; set; } } + +#pragma warning restore IDE1006 // Naming Styles diff --git a/test/Altinn.App.Api.Tests/Data/apps/tdd/contributer-restriction/ui/layouts/page.json b/test/Altinn.App.Api.Tests/Data/apps/tdd/contributer-restriction/ui/layouts/page.json index ca66ac17f..4d7d98ae9 100644 --- a/test/Altinn.App.Api.Tests/Data/apps/tdd/contributer-restriction/ui/layouts/page.json +++ b/test/Altinn.App.Api.Tests/Data/apps/tdd/contributer-restriction/ui/layouts/page.json @@ -17,7 +17,15 @@ "dataModelBindings": { "simpleBinding": "melding.name" } + }, + { + "id": "hidden", + "type": "Input", + "hidden": true, + "dataModelBindings": { + "simpleBinding": "melding.hidden" + } } ] } -} \ No newline at end of file +} diff --git a/test/Altinn.App.Core.Tests/Internal/Process/ProcessNavigatorTests.cs b/test/Altinn.App.Core.Tests/Internal/Process/ProcessNavigatorTests.cs index 9f6ead5a4..74b947fe9 100644 --- a/test/Altinn.App.Core.Tests/Internal/Process/ProcessNavigatorTests.cs +++ b/test/Altinn.App.Core.Tests/Internal/Process/ProcessNavigatorTests.cs @@ -200,7 +200,7 @@ public async Task GetNextTask_returns_empty_list_if_element_has_no_next() nextElements.Should().BeNull(); } - private IProcessNavigator SetupProcessNavigator( + private ProcessNavigator SetupProcessNavigator( string bpmnfile, IEnumerable gatewayFilters )