diff --git a/build/common.props b/build/common.props index 7d19eba7..1deb72f6 100644 --- a/build/common.props +++ b/build/common.props @@ -3,7 +3,7 @@ 3.0.0$(VersionSuffix) 5.0.0$(VersionSuffix) - 4.6.0$(VersionSuffix) + 4.6.1$(VersionSuffix) 3.2.0$(VersionSuffix) 3.0.0$(VersionSuffix) 3.0.3$(VersionSuffix) diff --git a/src/WebJobs.Extensions.CosmosDB/Trigger/CosmosDBMetricsProvider.cs b/src/WebJobs.Extensions.CosmosDB/Trigger/CosmosDBMetricsProvider.cs index 400e5f6c..ac01c375 100644 --- a/src/WebJobs.Extensions.CosmosDB/Trigger/CosmosDBMetricsProvider.cs +++ b/src/WebJobs.Extensions.CosmosDB/Trigger/CosmosDBMetricsProvider.cs @@ -17,6 +17,8 @@ internal class CosmosDBMetricsProvider private readonly Container _monitoredContainer; private readonly Container _leaseContainer; private readonly string _processorName; + private readonly int _maxAssignWorkerOnNotFoundCount = 5; + private int _assignWorkerOnNotFoundCount = 0; private static readonly Dictionary KnownDocumentClientErrors = new Dictionary() { @@ -59,7 +61,16 @@ public async Task GetMetricsAsync() } partitionCount = partitionWorkList.Count; - remainingWork = partitionWorkList.Sum(item => item.EstimatedLag); + if (partitionCount == 0) + { + partitionCount = 1; + remainingWork = 1; + _logger.LogWarning(Events.OnScaling, "PartitionCount is 0, the lease container exists but it has not been initialized, scale out to 1 and wait for the first execution."); + } + else + { + remainingWork = partitionWorkList.Sum(item => item.EstimatedLag); + } } catch (CosmosException cosmosException) when (cosmosException.StatusCode == HttpStatusCode.Gone) { @@ -69,6 +80,19 @@ public async Task GetMetricsAsync() partitionCount = 1; remainingWork = 1; } + catch (CosmosException cosmosException) when (cosmosException.StatusCode == HttpStatusCode.NotFound + && _assignWorkerOnNotFoundCount < _maxAssignWorkerOnNotFoundCount) + { + // An exception "Not found" may indicate that the lease container does not exist. + // We vote to scale out, assign a worker to create the lease container. + // However, it could also signal an issue with the monitoring container configuration. + // As a result, we make a limited number of attempts to create the lease container. + _assignWorkerOnNotFoundCount++; + _logger.LogWarning(Events.OnScaling, $"Possible non-exiting lease container detected. Trying to create the lease container, attempt '{_assignWorkerOnNotFoundCount}'", + cosmosException.GetType().ToString(), cosmosException.Message); + partitionCount = 1; + remainingWork = 1; + } catch (Exception e) when (e is CosmosException || e is InvalidOperationException) { if (!TryHandleCosmosException(e)) diff --git a/test/WebJobs.Extensions.CosmosDB.Tests/Trigger/CosmosDBMetricsProviderTests.cs b/test/WebJobs.Extensions.CosmosDB.Tests/Trigger/CosmosDBMetricsProviderTests.cs index c823e23b..ad8b0d4a 100644 --- a/test/WebJobs.Extensions.CosmosDB.Tests/Trigger/CosmosDBMetricsProviderTests.cs +++ b/test/WebJobs.Extensions.CosmosDB.Tests/Trigger/CosmosDBMetricsProviderTests.cs @@ -78,8 +78,8 @@ public async Task GetMetrics_ReturnsExpectedResult() var metrics = await _cosmosDbMetricsProvider.GetMetricsAsync(); - Assert.Equal(0, metrics.PartitionCount); - Assert.Equal(0, metrics.RemainingWork); + Assert.Equal(1, metrics.PartitionCount); + Assert.Equal(1, metrics.RemainingWork); Assert.NotEqual(default(DateTime), metrics.Timestamp); _estimatorIterator @@ -140,12 +140,12 @@ public async Task GetMetrics_HandlesExceptions() var metrics = (CosmosDBTriggerMetrics)await _cosmosDbMetricsProvider.GetMetricsAsync(); - Assert.Equal(0, metrics.PartitionCount); - Assert.Equal(0, metrics.RemainingWork); + Assert.Equal(1, metrics.PartitionCount); + Assert.Equal(1, metrics.RemainingWork); Assert.NotEqual(default(DateTime), metrics.Timestamp); var warning = _loggerProvider.GetAllLogMessages().Single(p => p.Level == Microsoft.Extensions.Logging.LogLevel.Warning); - Assert.Equal("Please check that the CosmosDB container and leases container exist and are listed correctly in Functions config files.", warning.FormattedMessage); + Assert.StartsWith("Possible non-exiting lease container detected. Trying to create the lease container, attempt", warning.FormattedMessage); _loggerProvider.ClearAllLogMessages(); await Assert.ThrowsAsync(async () => await _cosmosDbMetricsProvider.GetMetricsAsync());