Skip to content

Commit

Permalink
changed LengthFunctionDelegate to Func<string, int>
Browse files Browse the repository at this point in the history
  • Loading branch information
TesAnti committed Oct 18, 2023
1 parent b3152fe commit 9c340d4
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 4 deletions.
13 changes: 13 additions & 0 deletions src/libs/LangChain.Core/Base/BaseLoader.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
using LangChain.Docstore;

namespace LangChain.Base;

public abstract class BaseLoader
{
public abstract List<Document> Load();

public List<Document> LoadAndSplit(TextSplitter textSplitter=null)
{
throw new NotImplementedException();
}
}
8 changes: 4 additions & 4 deletions src/libs/LangChain.Core/Base/TextSplitter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ public abstract class TextSplitter
{
private readonly int _chunkSize;
private readonly int _chunkOverlap;
private readonly LengthFunctionDelegate _lengthFunction;
private readonly Func<string, int> _lengthFunction;

public delegate int LengthFunctionDelegate(string str);


protected TextSplitter(int chunkSize = 4000, int chunkOverlap = 200, LengthFunctionDelegate? lengthFunction = null)
protected TextSplitter(int chunkSize = 4000, int chunkOverlap = 200, Func<string,int>? lengthFunction = null)
{
if (chunkOverlap > chunkSize)
{
Expand All @@ -26,7 +26,7 @@ protected TextSplitter(int chunkSize = 4000, int chunkOverlap = 200, LengthFunct

_chunkSize = chunkSize;
_chunkOverlap = chunkOverlap;
_lengthFunction = lengthFunction ?? new LengthFunctionDelegate((str) => str.Length);
_lengthFunction = lengthFunction ?? new Func<string, int>((str) => str.Length);
}

public abstract List<string> SplitText(string text);
Expand Down

0 comments on commit 9c340d4

Please sign in to comment.