Skip to content

Commit

Permalink
refactor: Renamed CollectMetadata to CollectMetadataIfRequired.
Browse files Browse the repository at this point in the history
  • Loading branch information
HavenDV committed May 3, 2024
1 parent a07e22c commit c38570f
Show file tree
Hide file tree
Showing 6 changed files with 12 additions and 12 deletions.
8 changes: 4 additions & 4 deletions src/DocumentLoaders/Abstractions/src/DataSourceExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public static IReadOnlyDictionary<string, object> ToMetadata(this DataSource dat
/// <param name="settings"></param>
/// <param name="dataSource"></param>
/// <returns></returns>
public static IReadOnlyDictionary<string, object>? CollectMetadata(
public static IReadOnlyDictionary<string, object>? CollectMetadataIfRequired(
this DocumentLoaderSettings? settings,
DataSource dataSource)
{
Expand All @@ -43,11 +43,11 @@ public static IReadOnlyDictionary<string, object> ToMetadata(this DataSource dat
/// <param name="metadata"></param>
/// <param name="additionalMetadata"></param>
/// <returns></returns>
public static IReadOnlyDictionary<string, object>? With(
this IReadOnlyDictionary<string, object>? metadata,
public static IReadOnlyDictionary<string, object> With(
this IReadOnlyDictionary<string, object> metadata,
IReadOnlyDictionary<string, object> additionalMetadata)
{
return metadata?
return metadata
.Concat(additionalMetadata)
.ToDictionary(pair => pair.Key, pair => pair.Value);
}
Expand Down
4 changes: 2 additions & 2 deletions src/DocumentLoaders/Abstractions/src/FileLoader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ public async Task<IReadOnlyCollection<Document>> LoadAsync(
// }
// }

var metadata = settings.CollectMetadata(dataSource);

var metadata = settings.CollectMetadataIfRequired(dataSource);
return [new Document(content, metadata: metadata)];
}
}
4 changes: 2 additions & 2 deletions src/DocumentLoaders/Pdf/src/AsposePdfSource.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ public async Task<IReadOnlyCollection<Document>> LoadAsync(
var textAbsorber = new TextAbsorber();
pdfDocument.Pages.Accept(textAbsorber);

var metadata = settings.CollectMetadata(dataSource);

var metadata = settings.CollectMetadataIfRequired(dataSource);
return [new Document(textAbsorber.Text, metadata: metadata)];
}
}
4 changes: 2 additions & 2 deletions src/DocumentLoaders/Pdf/src/PdfPigPdfSource.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ public async Task<IReadOnlyCollection<Document>> LoadAsync(
using var stream = await dataSource.GetStreamAsync(cancellationToken).ConfigureAwait(false);
using var document = PdfDocument.Open(stream, new ParsingOptions());

var metadata = settings.CollectMetadata(dataSource);
var metadata = settings.CollectMetadataIfRequired(dataSource);

return document
.GetPages()
.Select(page => new Document(page.Text, metadata.With(new Dictionary<string, object>
.Select(page => new Document(page.Text, metadata?.With(new Dictionary<string, object>
{
{ "page", page.Number },
})))
Expand Down
2 changes: 1 addition & 1 deletion src/DocumentLoaders/WebBase/src/HtmlLoader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public async Task<IReadOnlyCollection<Document>> LoadAsync(
document.QuerySelector("html") ??
throw new NotSupportedException("Not supported for pages without <html> tag");

var metadata = settings.CollectMetadata(dataSource);
var metadata = settings.CollectMetadataIfRequired(dataSource);

return [new Document(html.TextContent, metadata: metadata)];
}
Expand Down
2 changes: 1 addition & 1 deletion src/DocumentLoaders/Word/src/WordLoader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public async Task<IReadOnlyCollection<Document>> LoadAsync(
}
}

var metadata = settings.CollectMetadata(dataSource);
var metadata = settings.CollectMetadataIfRequired(dataSource);

return documents
.Select(text => new Document(text, metadata: metadata))
Expand Down

0 comments on commit c38570f

Please sign in to comment.