Implement Activity State Filtering and JavaScript Integration (#5993)

* Add secret scripting integration for JavaScript

Introduced a new `Elsa.Secrets.Scripting` module that provides secret management capabilities within JavaScript workflows. This includes configuring the Jint engine to use workflow variables, adding new type and variable definition providers, and integrating with existing secret management features.

* Refactor secret name extraction to a separate method

Moved the logic for extracting secret names from the main method to a dedicated private method `GetSecretNamesFromExpression`. This improves code readability and maintains the single responsibility principle by delegating secret name extraction to its own method.

* Add input evaluation, sensitive input handling, and middleware refactor

Introduced methods for evaluating activity input properties and handling inputs marked as sensitive. Refactored `ExecutionLogMiddleware` constructor for consistency. Enhanced `SendHttpRequestBase` to mark authorization inputs as potentially containing secrets. Removed obsolete entries and adjusted persistence logic for clarity.

* Refactor IActivityStateProtector interface

Remove unused using directives and unnecessary comments. Simplify the definition of the `ProtectedActivityStateContext` record.

* Add activity state filtering mechanism

Introduce an abstract filter base class, context, and result models to enable filtering of activity state. Implement a default filter manager to run these filters and apply a specific filter for obfuscating HTTP request headers. Update necessary dependencies and extension methods to integrate the new filtering functionality.

* Add expired secrets management

Implemented services to manage expired secrets by periodically checking and updating their status. Introduced a new hosted service to perform the sweep and configurable options for the sweep interval. Updated related classes and configurations accordingly.

* Update SweepInterval in appsettings.json

Changed the Secrets Management SweepInterval from 30 seconds to 4 hours. This adjustment aims to reduce the frequency of sweep operations and improve overall system performance.

* Update comment to reflect configuring engine with secrets

The comment was changed to better describe the handler's function, specifying that it configures the Jint engine with secrets instead of workflow variables. This clarifies the purpose and usage of the handler in the context of the code.

* Remove unused inputDescriptors variable

This commit removes the inputDescriptors variable, which was declared but never used in DefaultActivityExecutionMapper.cs. This helps in cleaning up the code and potentially reducing memory usage. Ensuring that all declared variables are utilized can improve code readability and maintainability.

Directory.Packages.props

@@ -95,7 +95,6 @@
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="System.Linq.Dynamic.Core" Version="1.4.3" />
     <PackageVersion Include="System.Net.Http" Version="4.3.4" />
-    <PackageVersion Include="System.Text.RegularExpressions" Version="4.3.1" />
     <PackageVersion Include="Testcontainers" Version="3.9.0" />
     <PackageVersion Include="Testcontainers.RabbitMq" Version="3.9.0" />
     <PackageVersion Include="Testcontainers.Redis" Version="3.9.0" />

Elsa.sln

@@ -466,6 +466,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elsa.Agents.Persistence.Ent
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elsa.Secrets.Models", "src\modules\Elsa.Secrets.Models\Elsa.Secrets.Models.csproj", "{29D12ADC-55E9-40D0-9E4C-F0EBB6E098EC}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elsa.Secrets.Scripting", "src\modules\Elsa.Secrets.Scripting\Elsa.Secrets.Scripting.csproj", "{6C606FEB-9A1F-4816-ABE4-22AFA8CEE771}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -1058,6 +1060,10 @@ Global
 		{29D12ADC-55E9-40D0-9E4C-F0EBB6E098EC}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{29D12ADC-55E9-40D0-9E4C-F0EBB6E098EC}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{29D12ADC-55E9-40D0-9E4C-F0EBB6E098EC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6C606FEB-9A1F-4816-ABE4-22AFA8CEE771}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6C606FEB-9A1F-4816-ABE4-22AFA8CEE771}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6C606FEB-9A1F-4816-ABE4-22AFA8CEE771}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6C606FEB-9A1F-4816-ABE4-22AFA8CEE771}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1243,6 +1249,7 @@ Global
 		{A9140976-DFF9-432A-B1DC-E188A07C49E5} = {50470834-4CD8-479A-8B58-0A1869BA5D37}
 		{B3046301-6F00-4885-8B01-080BD489055C} = {50470834-4CD8-479A-8B58-0A1869BA5D37}
 		{29D12ADC-55E9-40D0-9E4C-F0EBB6E098EC} = {8CEEC194-820A-4C8D-AB9E-E51E6D3E9CC1}
+		{6C606FEB-9A1F-4816-ABE4-22AFA8CEE771} = {8CEEC194-820A-4C8D-AB9E-E51E6D3E9CC1}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {D4B5CEAA-7D70-4FCB-A68E-B03FBE5E0E5E}

src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj

@@ -12,6 +12,7 @@
         <ProjectReference Include="..\..\modules\Elsa.OpenTelemetry\Elsa.OpenTelemetry.csproj" />
         <ProjectReference Include="..\..\modules\Elsa.Secrets.Api\Elsa.Secrets.Api.csproj" />
         <ProjectReference Include="..\..\modules\Elsa.Secrets.Persistence.EntityFrameworkCore.Sqlite\Elsa.Secrets.Persistence.EntityFrameworkCore.Sqlite.csproj" />
+        <ProjectReference Include="..\..\modules\Elsa.Secrets.Scripting\Elsa.Secrets.Scripting.csproj" />
         <ProjectReference Include="..\..\modules\Elsa\Elsa.csproj"/>
         <ProjectReference Include="..\..\common\Elsa.DropIns\Elsa.DropIns.csproj"/>
         <ProjectReference Include="..\..\modules\Elsa.Alterations.MassTransit\Elsa.Alterations.MassTransit.csproj"/>

src/apps/Elsa.Server.Web/Filters/HttpRequestAuthenticationHeaderFilter.cs

@@ -0,0 +1,30 @@
+using Elsa.Http;
+using Elsa.Workflows;
+using JetBrains.Annotations;
+
+namespace Elsa.Server.Web.Filters;
+
+/// <summary>
+/// Mask the value of the input if it is a HttpRequest and the input name is "Authorization".
+/// </summary>
+[UsedImplicitly]
+public class HttpRequestAuthenticationHeaderFilter : ActivityStateFilterBase
+{
+    protected override ActivityStateFilterResult OnExecute(ActivityStateFilterContext context)
+    {
+        var activityExecutionContext = context.ActivityExecutionContext;
+        var activity = activityExecutionContext.Activity;
+        var inputDescriptor = context.InputDescriptor;
+
+        if (activity is not SendHttpRequestBase || inputDescriptor.Name is not nameof(SendHttpRequestBase.Authorization))
+            return ActivityStateFilterResult.Pass();
+
+        var contextValue = context.Value.GetString();
+
+        if (contextValue == null)
+            return ActivityStateFilterResult.Pass();
+
+        var maskedValue = new string('*', contextValue.Length);
+        return Filtered(maskedValue);
+    }
+}

src/apps/Elsa.Server.Web/Program.cs

@@ -25,6 +25,7 @@
 using Elsa.Secrets.Extensions;
 using Elsa.Secrets.Persistence;
 using Elsa.Server.Web;
+using Elsa.Server.Web.Filters;
 using Elsa.Tenants.Extensions;
 using Elsa.Workflows;
 using Elsa.Workflows.Api;
@@ -436,8 +437,13 @@
         {
             elsa
                 .UseSecrets()
-                .UseSecretsManagement(management => management.UseEntityFrameworkCore(ef => ef.UseSqlite(sqliteConnectionString)))
+                .UseSecretsManagement(management =>
+                {
+                    management.ConfigureOptions(options => configuration.GetSection("Secrets:Management").Bind(options));
+                    management.UseEntityFrameworkCore(ef => ef.UseSqlite(sqliteConnectionString));
+                })
                 .UseSecretsApi()
+                .UseSecretsScripting()
                 ;
         }
 
@@ -463,6 +469,9 @@
         ConfigureForTest?.Invoke(elsa);
     });
 
+// Obfuscate HTTP request headers.
+services.AddActivityStateFilter<HttpRequestAuthenticationHeaderFilter>();
+
 //services.Configure<CachingOptions>(options => options.CacheDuration = TimeSpan.FromDays(1));
 services.AddHealthChecks();
 services.AddControllers();

src/apps/Elsa.Server.Web/appsettings.json

@@ -173,6 +173,11 @@
       }
     ]
   },
+  "Secrets": {
+    "Management": {
+      "SweepInterval": "04:00:00"
+    }
+  },
   "Agents": {
     "ApiKeys": [
       {

src/modules/Elsa.Http/Activities/SendHttpRequestBase.cs

@@ -15,18 +15,12 @@ namespace Elsa.Http;
 /// Base class for activities that send HTTP requests.
 /// </summary>
 [Output(IsSerializable = false)]
-public abstract class SendHttpRequestBase : Activity<HttpResponseMessage>
+public abstract class SendHttpRequestBase(string? source = default, int? line = default) : Activity<HttpResponseMessage>(source, line)
 {
-    /// <inheritdoc />
-    protected SendHttpRequestBase(string? source = default, int? line = default) : base(source, line)
-    {
-    }
-
     /// <summary>
     /// The URL to send the request to.
     /// </summary>
-    [Input]
-    public Input<Uri?> Url { get; set; } = default!;
+    [Input] public Input<Uri?> Url { get; set; } = default!;
 
     /// <summary>
     /// The HTTP method to use when sending the request.
@@ -62,7 +56,10 @@ protected SendHttpRequestBase(string? source = default, int? line = default) : b
     /// The Authorization header value to send with the request.
     /// </summary>
     /// <example>Bearer {some-access-token}</example>
-    [Input(Description = "The Authorization header value to send with the request. For example: Bearer {some-access-token}", Category = "Security")]
+    [Input(
+        Description = "The Authorization header value to send with the request. For example: Bearer {some-access-token}",
+        Category = "Security",
+        CanContainSecrets = true)]
     public Input<string?> Authorization { get; set; } = default!;
 
     /// <summary>

src/modules/Elsa.JavaScript/Notifications/EvaluatedJavaScript.cs

@@ -7,4 +7,4 @@ namespace Elsa.JavaScript.Notifications;
 /// <summary>
 /// This notification is published every time a JavaScript expression has been evaluated.
 /// </summary>
-public record EvaluatedJavaScript(Engine Engine, ExpressionExecutionContext Context, object? Result) : INotification;
+public record EvaluatedJavaScript(Engine Engine, ExpressionExecutionContext Context, string Expression, object? Result) : INotification;

src/modules/Elsa.JavaScript/Notifications/EvaluatingJavaScript.cs

@@ -8,4 +8,4 @@ namespace Elsa.JavaScript.Notifications;
 /// This notification is published every time a JavaScript expression is about to be evaluated.
 /// It gives subscribers a chance to configure the <see cref="Engine"/> with additional functions and variables.
 /// </summary>
-public record EvaluatingJavaScript(Engine Engine, ExpressionExecutionContext Context) : INotification;
+public record EvaluatingJavaScript(Engine Engine, ExpressionExecutionContext Context, string Expression) : INotification;

src/modules/Elsa.JavaScript/Services/JintJavaScriptEvaluator.cs

@@ -37,9 +37,9 @@ public class JintJavaScriptEvaluator(IConfiguration configuration, INotification
         CancellationToken cancellationToken = default)
     {
         var engine = await GetConfiguredEngine(configureEngine, context, options, cancellationToken);
-        await mediator.SendAsync(new EvaluatingJavaScript(engine, context), cancellationToken);
+        await mediator.SendAsync(new EvaluatingJavaScript(engine, context, expression), cancellationToken);
         var result = ExecuteExpressionAndGetResult(engine, expression);
-        await mediator.SendAsync(new EvaluatedJavaScript(engine, context, result), cancellationToken);
+        await mediator.SendAsync(new EvaluatedJavaScript(engine, context, expression, result), cancellationToken);
 
         return result.ConvertTo(returnType);
     }

src/modules/Elsa.JavaScript/TypeDefinitions/Abstractions/VariableDefinitionProvider.cs

@@ -23,6 +23,6 @@ protected VariableDefinition CreateVariableDefinition(Action<VariableDefinitionB
     {
         var builder = new VariableDefinitionBuilder();
         setup(builder);
-        return builder.BuildVariableDefinition();
+        return builder.Build();
     }
 }

src/modules/Elsa.JavaScript/TypeDefinitions/Builders/VariableDefinitionBuilder.cs

@@ -18,5 +18,5 @@ public VariableDefinitionBuilder Type(string type)
         return this;
     }
 
-    public VariableDefinition BuildVariableDefinition() => new(_variableDefinition.Name, _variableDefinition.Type);
+    public VariableDefinition Build() => new(_variableDefinition.Name, _variableDefinition.Type);
 }

src/modules/Elsa.Secrets.Api/Endpoints/Secrets/Update/Endpoint.cs

@@ -28,7 +28,7 @@ public override async Task<SecretModel> ExecuteAsync(SecretInputModel req, Cance
             return null!;
         }
 
-        var isNameDuplicate = !await nameValidator.IsNameUniqueAsync(req.Name, id, ct);
+        var isNameDuplicate = !await nameValidator.IsNameUniqueAsync(req.Name, entity.SecretId, ct);
 
         if (isNameDuplicate)
         {

src/modules/Elsa.Secrets.Management/Contracts/IExpiredSecretsUpdater.cs

@@ -0,0 +1,6 @@
+namespace Elsa.Secrets.Management;
+
+public interface IExpiredSecretsUpdater
+{
+    Task UpdateExpiredSecretsAsync(CancellationToken cancellationToken = default);
+}

src/modules/Elsa.Secrets.Management/Contracts/ISecretManager.cs

@@ -16,6 +16,9 @@ public interface ISecretManager
     /// Finds the entity from the store.
     Task<Secret?> FindAsync(SecretFilter filter, CancellationToken cancellationToken = default);
 
+    /// Finds all entities from the store matching the specified filter.
+    Task<IEnumerable<Secret>> FindManyAsync(SecretFilter filter, CancellationToken cancellationToken = default);
+
     /// Gets all entities from the store.
     Task<IEnumerable<Secret>> ListAsync(CancellationToken cancellationToken = default);
 

src/modules/Elsa.Secrets.Management/Features/SecretManagementFeature.cs

@@ -4,6 +4,7 @@
 using Elsa.Features.Services;
 using Elsa.Secrets.Extensions;
 using Elsa.Secrets.Features;
+using Elsa.Secrets.Management.HostedService;
 using Microsoft.Extensions.DependencyInjection;
 
 namespace Elsa.Secrets.Management.Features;
@@ -20,6 +21,12 @@ public SecretManagementFeature UseSecretsStore(Func<IServiceProvider, ISecretSto
         _secretStoreFactory = secretStoreFactory;
         return this;
     }
+
+    public SecretManagementFeature ConfigureOptions(Action<SecretManagementOptions> configureOptions)
+    {
+        Services.Configure(configureOptions);
+        return this;
+    }
 
     public override void Configure()
     {
@@ -29,6 +36,11 @@ public override void Configure()
         });
     }
 
+    public override void ConfigureHostedServices()
+    {
+        ConfigureHostedService<ExpiredSecretsHostedService>();
+    }
+
     public override void Apply()
     {
         Services
@@ -43,6 +55,7 @@ public override void Apply()
             .AddScoped<ISecretNameValidator, DefaultSecretNameValidator>()
             .AddScoped<ISecretUpdater, DefaultSecretUpdater>()
             .AddScoped<ISecretManager, DefaultSecretManager>()
+            .AddScoped<IExpiredSecretsUpdater, DefaultExpiredSecretsUpdater>()
             ;
     }
 }

src/modules/Elsa.Secrets.Management/HostedService/ExpiredSecretsHostedService.cs

@@ -0,0 +1,54 @@
+using JetBrains.Annotations;
+using Medallion.Threading;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Elsa.Secrets.Management.HostedService;
+
+[UsedImplicitly]
+public class ExpiredSecretsHostedService(IOptions<SecretManagementOptions> options, IDistributedLockProvider distributedLockProvider, IServiceScopeFactory scopeFactory, ILogger<ExpiredSecretsHostedService> logger) : BackgroundService
+{
+    private Timer _timer = default!;
+
+    protected override Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        // Get the configured sweep interval from the options, and use it to periodically sweep expired secrets.
+        var sweepInterval = options.Value.SweepInterval;
+
+        // Set up a timer that will sweep expired secrets at the configured interval.
+        _timer = new Timer(SweepExpiredSecrets, null, sweepInterval, sweepInterval);
+
+        return Task.CompletedTask;
+    }
+
+    public override Task StopAsync(CancellationToken cancellationToken)
+    {
+        _timer.Change(Timeout.Infinite, 0);
+        _timer.Dispose();
+        return base.StopAsync(cancellationToken);
+    }
+
+    private async void SweepExpiredSecrets(object? state)
+    {
+        // Acquire a distributed lock to ensure that only one instance of the hosted service is running at any given time.
+        await using var distributedLock = await distributedLockProvider.TryAcquireLockAsync("expired-secrets-sweep");
+
+        // If the lock could not be acquired, return a completed task.
+        if (distributedLock == null)
+        {
+            logger.LogInformation("Another instance of the expired secrets hosted service is already running. Exiting...");
+            return;
+        }
+
+        // Sweep expired secrets here.
+        logger.LogInformation("Sweeping expired secrets...");
+
+        using var scope = scopeFactory.CreateScope();
+        var updater = scope.ServiceProvider.GetRequiredService<IExpiredSecretsUpdater>();
+        await updater.UpdateExpiredSecretsAsync();
+
+        logger.LogInformation("Expired secrets have been swept.");
+    }
+}

src/modules/Elsa.Secrets.Management/Models/SecretFilter.cs

@@ -6,24 +6,30 @@ public class SecretFilter
     public ICollection<string>? Ids { get; set; }
     public string? SecretId { get; set; }
     public ICollection<string>? SecretIds { get; set; }
-    public string? NotId { get; set; }
+    public string? NotSecretId { get; set; }
     public string? Name { get; set; }
+    public ICollection<string>? Names { get; set; }
     public int? Version { get; set; }
     public string? Type { get; set; }
     public SecretStatus? Status { get; set; }
     public bool IsLatest { get; set; }
     public string? SearchTerm { get; set; }
+    public DateTimeOffset? ExpiresAtLessThan { get; set; }
 
     public IQueryable<Secret> Apply(IQueryable<Secret> queryable)
     {
         if (Id != null) queryable = queryable.Where(x => x.Id == Id);
         if (Ids != null) queryable = queryable.Where(x => Ids.Contains(x.Id));
-        if (NotId != null) queryable = queryable.Where(x => x.Id != NotId);
+        if (SecretId != null) queryable = queryable.Where(x => x.SecretId == SecretId);
+        if (SecretIds != null) queryable = queryable.Where(x => SecretIds.Contains(x.SecretId));
+        if (NotSecretId != null) queryable = queryable.Where(x => x.SecretId != NotSecretId);
         if (Name != null) queryable = queryable.Where(x => x.Name == Name);
+        if (Names != null) queryable = queryable.Where(x => Names.Contains(x.Name));
         if (Version != null) queryable = queryable.Where(x => x.Version == Version);
         if (Type != null) queryable = queryable.Where(x => x.Scope == Type);
         if(Status != null) queryable = queryable.Where(x => x.Status == Status);
         if (IsLatest) queryable = queryable.Where(x => x.IsLatest);
+        if (ExpiresAtLessThan != null) queryable = queryable.Where(x => x.ExpiresAt < ExpiresAtLessThan);
         if (!string.IsNullOrWhiteSpace(SearchTerm)) queryable = queryable.Where(x => x.Name.Contains(SearchTerm) || x.Description.Contains(SearchTerm) || x.Id.Contains(SearchTerm));
 
         return queryable;

src/modules/Elsa.Secrets.Management/Options/SecretManagementOptions.cs

@@ -0,0 +1,9 @@
+namespace Elsa.Secrets.Management;
+
+public class SecretManagementOptions
+{
+    /// <summary>
+    /// The interval at which the background sweep should run for expired secrets.
+    /// </summary>
+    public TimeSpan SweepInterval { get; set; } = TimeSpan.FromHours(12);
+}