Skip to content

Commit

Permalink
on multigpu setups, pick lowest free mem instead of highest for auto …
Browse files Browse the repository at this point in the history 
…layers
  • Loading branch information
@LostRuins
LostRuins committed Aug 20, 2024
1 parent 3bd70d7 commit 7ee359a
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 21 deletions.
96 changes: 80 additions & 16 deletions klite.embd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ Current version indicated by LITEVER below.
-->

<script>
const LITEVER = 165;
const LITEVER = 167;
const urlParams = new URLSearchParams(window.location.search);
const localflag = true;
const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
Expand Down Expand Up @@ -734,6 +734,9 @@ Current version indicated by LITEVER below.
.color_gray {
color: #9b9b9b;
}
.color_lightgray {
color: #bbbbbb;
}
.color_red {
color: #ff7967;
}
Expand Down Expand Up @@ -4266,6 +4269,7 @@ Current version indicated by LITEVER below.
opmode: 4, //what mode are we in? 1=story, 2=adventure, 3=chat, 4=instruct
adventure_is_action: false, //in adventure mode, determine story or action
adventure_context_mod: true, //extra injection for adventure mode
fix_alpaca_leak: true, //prevents leaking when Alpaca instruct format is used on crappy models
chat_context_mod: true, //extra injection for chat mode
chatname: "User", //name to use in chat
chatopponent: defaultchatopponent,
Expand Down Expand Up @@ -4293,7 +4297,8 @@ Current version indicated by LITEVER below.
img_autogen: false,
img_allownsfw: true,
img_cfgscale: 7,
img_allowhd: false,
img_allowhd: true,
img_crop: false,
img_img2imgstr: 0.6,
img_steps: 20,
img_sampler: "Euler a",
Expand Down Expand Up @@ -9603,6 +9608,7 @@ Current version indicated by LITEVER below.
document.getElementById("inject_jailbreak_instruct").checked = localsettings.inject_jailbreak_instruct;
document.getElementById("idle_responses").value = localsettings.idle_responses;
document.getElementById("idle_duration").value = localsettings.idle_duration;
document.getElementById("fix_alpaca_leak").checked = localsettings.fix_alpaca_leak;
document.getElementById("adventure_context_mod").checked = localsettings.adventure_context_mod;
document.getElementById("chat_context_mod").checked = localsettings.chat_context_mod;
document.getElementById("instruct_has_markdown").checked = localsettings.instruct_has_markdown;
Expand Down Expand Up @@ -9674,6 +9680,7 @@ Current version indicated by LITEVER below.

document.getElementById("tokenstreammode").value = localsettings.tokenstreammode;
document.getElementById("img_allowhd").checked = localsettings.img_allowhd;
document.getElementById("img_crop").checked = localsettings.img_crop;
document.getElementById("img_autogen").checked = localsettings.img_autogen;
document.getElementById("save_images").checked = localsettings.save_images;
document.getElementById("save_remote_images").checked = localsettings.save_remote_images;
Expand Down Expand Up @@ -9915,6 +9922,7 @@ Current version indicated by LITEVER below.
localsettings.inject_jailbreak_instruct = (document.getElementById("inject_jailbreak_instruct").checked ? true : false);
localsettings.idle_responses = document.getElementById("idle_responses").value;
localsettings.idle_duration = document.getElementById("idle_duration").value;
localsettings.fix_alpaca_leak = (document.getElementById("fix_alpaca_leak").checked ? true : false);
localsettings.adventure_context_mod = (document.getElementById("adventure_context_mod").checked ? true : false);
localsettings.chat_context_mod = (document.getElementById("chat_context_mod").checked ? true : false);
localsettings.instruct_has_markdown = (document.getElementById("instruct_has_markdown").checked ? true : false);
Expand Down Expand Up @@ -9960,6 +9968,7 @@ Current version indicated by LITEVER below.
localsettings.image_negprompt = document.getElementById("negpromptinput").value;
localsettings.grammar = pendinggrammar;
localsettings.tokenstreammode = document.getElementById("tokenstreammode").value;
localsettings.img_crop = (document.getElementById("img_crop").checked ? true : false);
localsettings.img_allowhd = (document.getElementById("img_allowhd").checked ? true : false);
localsettings.img_autogen = (document.getElementById("img_autogen").checked ? true : false);
localsettings.save_images = (document.getElementById("save_images").checked ? true : false);
Expand Down Expand Up @@ -11965,6 +11974,10 @@ Current version indicated by LITEVER below.
let st = get_instruct_starttag(true);
let et = get_instruct_endtag(true);
seqs = [st, et];
if(localsettings.fix_alpaca_leak && st.toLowerCase().includes("### instruction"))
{
seqs.push("\n\n### ");
}
if(localsettings.inject_chatnames_instruct)
{
if(localsettings.chatname!="")
Expand Down Expand Up @@ -13611,7 +13624,7 @@ Current version indicated by LITEVER below.
}
}

function compressImage(inputDataUri, onDone, isJpeg=true, fixedSize=true, maxSize=NO_HD_RES_PX, quality = 0.35, forceCrop=false) {
function compressImage(inputDataUri, onDone, isJpeg=true, fixedSize=true, maxSize=NO_HD_RES_PX, quality = 0.35, forceAspect=false) {
let img = document.createElement('img');
let wantedWidth = maxSize;
let wantedHeight = maxSize;
Expand Down Expand Up @@ -13645,7 +13658,7 @@ Current version indicated by LITEVER below.
canvas.height = wantedHeight;

// We resize the image with the canvas method
if(forceCrop)
if(forceAspect)
{
let minsizeW = Math.min(origW, origH);
let minsizeH = Math.min(origW, origH);
Expand All @@ -13671,9 +13684,26 @@ Current version indicated by LITEVER below.
canvas.height = wantedHeight = maxSize;
}

let mx = (origW - minsizeW) / 2;
let my = (origH - minsizeH) / 2;
ctx.drawImage(this, mx, my, minsizeW, minsizeH, 0, 0, wantedWidth, wantedHeight);
let newWidth, newHeight, mx, my;
if (wantedWidth / wantedHeight > aspectratio) {
newHeight = wantedHeight;
newWidth = wantedHeight * aspectratio;
} else {
newWidth = wantedWidth;
newHeight = wantedWidth / aspectratio;
}

if (localsettings.img_crop) {
mx = (origW - minsizeW) / 2;
my = (origH - minsizeH) / 2;
ctx.drawImage(this, mx, my, minsizeW, minsizeH, 0, 0, wantedWidth, wantedHeight);
} else {
mx = (wantedWidth - newWidth) / 2;
my = (wantedHeight - newHeight) / 2;
ctx.fillStyle = "black";
ctx.fillRect(0, 0, wantedWidth, wantedHeight);
ctx.drawImage(this, mx, my, newWidth, newHeight);
}
}else{
ctx.drawImage(this, 0, 0, wantedWidth, wantedHeight);
}
Expand Down Expand Up @@ -14244,16 +14274,25 @@ Current version indicated by LITEVER below.
if (isSupported) {
warn_on_quit = true;
const selection = window.getSelection();
if(selection.focusNode!=null && selection.focusNode.parentElement!=null
&& selection.focusNode.parentElement.classList.contains("txtchunk"))
let foundparent = null;
if (selection.focusNode != null && selection.focusNode.parentElement != null
&& selection.focusNode.parentElement.classList.contains("txtchunk")) {
foundparent = selection.focusNode.parentElement;
} else if (selection.focusNode != null && selection.focusNode.parentElement != null
&& selection.focusNode.parentElement.parentElement != null
&& selection.focusNode.parentElement.parentElement.classList.contains("txtchunk")) {
//double nested
foundparent = selection.focusNode.parentElement.parentElement;
}
if (foundparent)
{
if(prev_hl_chunk!=null)
{
if (prev_hl_chunk != null) {
prev_hl_chunk.classList.remove("hlchunk");
}
prev_hl_chunk = selection.focusNode.parentElement;
prev_hl_chunk = foundparent;
prev_hl_chunk.classList.add("hlchunk");
}

idle_timer = 0;
}
}
Expand Down Expand Up @@ -14481,7 +14520,7 @@ Current version indicated by LITEVER below.
fulltxt = fulltxt.replace(/\[<\|d\|.+?\|d\|>\]/g, stripimg_replace_str);

//always filter comments - new format
fulltxt = fulltxt.replace(/\[<\|.+?\|>\]/g, ""); //remove normal comments too
fulltxt = fulltxt.replace(/\[<\|[\s\S]+?\|>\]/g, ""); //remove normal comments too

}
return fulltxt;
Expand Down Expand Up @@ -15402,7 +15441,7 @@ Current version indicated by LITEVER below.
inner = render_image_html(inner, "",false,true);
return inner;
});
processed_msg = processed_msg.replace(/\[<\|.+?\|>\]/g, ""); //remove normal comments too
processed_msg = processed_msg.replace(/\[<\|[\s\S]+?\|>\]/g, ""); //remove normal comments too
}

let namepart = (curr.myturn ? "User" : cosmetic_corpo_ai_nick);
Expand Down Expand Up @@ -15430,6 +15469,16 @@ Current version indicated by LITEVER below.
}
}

let foundTimestamp = "";
if(localsettings.inject_timestamps)
{
let found = processed_msg.match(/(\[\d{1,2}\/\d{1,2}\/\d{4}, \d{1,2}:\d{2} [AP]M\]) /g);
if(found && found.length>0)
{
foundTimestamp = found[0];
processed_msg = processed_msg.replace(/(\[\d{1,2}\/\d{1,2}\/\d{4}, \d{1,2}:\d{2} [AP]M\]) /g, "");
}
}
for(let i=0;i<validprefixes.length;++i)
{
let person = validprefixes[i];
Expand All @@ -15441,6 +15490,10 @@ Current version indicated by LITEVER below.
break;
}
}
if(foundTimestamp)
{
processed_msg = foundTimestamp + "\n" + processed_msg;
}
}

let bodypart = (corpo_editing_turn == i ?
Expand Down Expand Up @@ -15579,7 +15632,7 @@ Current version indicated by LITEVER below.
for(var i=0;i<input.length;++i)
{
let tempfullsearchable = input[i]; //strip out images
let txtwithnoimages = tempfullsearchable.replace(/\[<\|.+?\|>\]/g, "");
let txtwithnoimages = tempfullsearchable.replace(/\[<\|[\s\S]+?\|>\]/g, "");
var foundopponent = txtwithnoimages.match(othernamesregex);
var foundself = txtwithnoimages.match(mynameregex);

Expand Down Expand Up @@ -15668,7 +15721,7 @@ Current version indicated by LITEVER below.
inner = render_image_html(inner, "",false,true);
return inner;
});
curr.msg = curr.msg.replace(/\[<\|.+?\|>\]/g, ""); //remove normal comments too
curr.msg = curr.msg.replace(/\[<\|[\s\S]+?\|>\]/g, ""); //remove normal comments too


}
Expand Down Expand Up @@ -17709,6 +17762,11 @@ Current version indicated by LITEVER below.
class="helptext">Modifies the context, injecting tokens to improve adventure quality for new adventures.</span></span> </div>
<input type="checkbox" title="Adventure PrePrompt" id="adventure_context_mod" style="margin:0px 0px 0px auto;">
</div>
<div class="settinglabel">
<div class="justifyleft settingsmall">Fix Alpaca Leakage <span class="helpicon">?<span
class="helptext">Prevents leaking when Alpaca instruct format is used on models trained with bad/different formats.</span></span> </div>
<input type="checkbox" title="Fix Alpaca Leakage" id="fix_alpaca_leak" style="margin:0px 0px 0px auto;">
</div>
</div>

<div class="settingitem">
Expand Down Expand Up @@ -18192,6 +18250,12 @@ Current version indicated by LITEVER below.
</span>: </div>
<input title="Save Higher-Res Images" type="checkbox" id="img_allowhd" style="margin:0px 0 0;">
</div>
<div class="inlinelabel">
<div class="justifyleft" style="padding:4px">Crop Images <span class="helpicon">?
<span class="helptext">If enabled, oversized imported images will be cropped to fit. If disabled, images will be letterboxed instead.</span>
</span>: </div>
<input title="Crop Images" type="checkbox" id="img_crop" style="margin:0px 0 0;">
</div>
</div>
</div>
</div>
Expand Down
14 changes: 9 additions & 5 deletions koboldcpp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -768,16 +768,20 @@ def fetch_gpu_properties(testCL,testCU,testVK):
FetchedCUdeviceMem = [line.split(",")[1].strip() for line in getamdvram.splitlines()[1:] if line.strip()]
except Exception as e:
pass
lowestcumem = 0
lowestfreecumem = 0
for idx in range(0,4):
if(len(FetchedCUdevices)>idx):
CUDevicesNames[idx] = FetchedCUdevices[idx]
if len(FetchedCUdeviceMem)>idx:
if AMDgpu:
MaxMemory[0] = max(int(FetchedCUdeviceMem[idx]),MaxMemory[0])
else:
MaxMemory[0] = max(int(FetchedCUdeviceMem[idx])*1024*1024,MaxMemory[0])
dmem = int(FetchedCUdeviceMem[idx]) if AMDgpu else (int(FetchedCUdeviceMem[idx])*1024*1024)
lowestcumem = dmem if lowestcumem==0 else (dmem if dmem<lowestcumem else lowestcumem)
if len(FetchedCUfreeMem)>idx:
MaxFreeMemory[0] = max(int(FetchedCUfreeMem[idx])*1024*1024,MaxFreeMemory[0])
dmem = (int(FetchedCUfreeMem[idx])*1024*1024)
lowestfreecumem = dmem if lowestfreecumem==0 else (dmem if dmem<lowestfreecumem else lowestfreecumem)

MaxMemory[0] = max(lowestcumem,MaxMemory[0])
MaxFreeMemory[0] = max(lowestfreecumem,MaxFreeMemory[0])

if testVK:
try: # Get Vulkan names
Expand Down

0 comments on commit 7ee359a

Please sign in to comment.