Skip to content

Commit

Permalink
improve build process
Browse files Browse the repository at this point in the history
  • Loading branch information
@amakropoulos
amakropoulos committed Jul 26, 2024
1 parent ee7c6ce commit 621ca4c
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 144 deletions.
142 changes: 25 additions & 117 deletions Editor/LLMBuildProcessor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,150 +2,58 @@
using UnityEditor.Build;
using UnityEditor.Build.Reporting;
using UnityEngine;
using System.IO;
using System.Collections.Generic;
using System;

namespace LLMUnity
{
public class LLMBuildProcessor : MonoBehaviour, IPreprocessBuildWithReport, IPostprocessBuildWithReport
public class LLMBuildProcessor : IPreprocessBuildWithReport, IPostprocessBuildWithReport
{
public int callbackOrder => 0;
static string tempDir = Path.Combine(Application.temporaryCachePath, "LLMBuildProcessor", Path.GetFileName(LLMUnitySetup.libraryPath));
static List<MovedPair> movedPairs = new List<MovedPair>();
static string movedCache = Path.Combine(tempDir, "moved.json");

[InitializeOnLoadMethod]
private static void InitializeOnLoad()
{
if (!Directory.Exists(tempDir)) Directory.CreateDirectory(tempDir);
else ResetMoves();
}

// CALLED BEFORE THE BUILD
// called before the build
public void OnPreprocessBuild(BuildReport report)
{
// Start listening for errors when build starts
Application.logMessageReceived += OnBuildError;
HideLibraryPlatforms(report.summary.platform);
HideModels();
if (movedPairs.Count > 0) AssetDatabase.Refresh();
}

// CALLED DURING BUILD TO CHECK FOR ERRORS
private void OnBuildError(string condition, string stacktrace, LogType type)
{
if (type == LogType.Error)
{
// FAILED TO BUILD, STOP LISTENING FOR ERRORS
BuildCompleted();
}
}

// CALLED AFTER THE BUILD
public void OnPostprocessBuild(BuildReport report)
{
BuildCompleted();
}

public void BuildCompleted()
{
Application.logMessageReceived -= OnBuildError;
ResetMoves();
}

static bool MovePath(string source, string target)
{
bool moved = false;
if (File.Exists(source))
{
File.Move(source, target);
moved = true;
}
else if (Directory.Exists(source))
{
Directory.Move(source, target);
moved = true;
}
if (moved)
{
movedPairs.Add(new MovedPair {source = source, target = target});
File.WriteAllText(movedCache, JsonUtility.ToJson(new FoldersMovedWrapper { movedPairs = movedPairs }));
}
return moved;
}

static void MoveAssetAndMeta(string source, string target)
{
MovePath(source + ".meta", target + ".meta");
MovePath(source, target);
}

static void HideLibraryPlatforms(BuildTarget buildPlatform)
{
List<string> platforms = new List<string>(){ "windows", "macos", "linux", "android" };
switch (buildPlatform)
string platform = null;
switch (report.summary.platform)
{
case BuildTarget.StandaloneWindows:
case BuildTarget.StandaloneWindows64:
platforms.Remove("windows");
platform = "windows";
break;
case BuildTarget.StandaloneLinux64:
platforms.Remove("linux");
platform = "linux";
break;
case BuildTarget.StandaloneOSX:
platforms.Remove("macos");
platform = "macos";
break;
case BuildTarget.Android:
platforms.Remove("android");
platform = "android";
break;
case BuildTarget.iOS:
platform = "ios";
break;
}

foreach (string dirname in Directory.GetDirectories(LLMUnitySetup.libraryPath))
{
foreach (string platform in platforms)
{
if (Path.GetFileName(dirname).StartsWith(platform))
{
MoveAssetAndMeta(dirname, Path.Combine(tempDir, Path.GetFileName(dirname)));
}
}
}
LLMBuilder.HideLibraryPlatforms(platform);
LLMBuilder.CopyModels();
AssetDatabase.Refresh();
}

static void HideModels()
// called during build to check for errors
private void OnBuildError(string condition, string stacktrace, LogType type)
{
foreach (LLM llm in FindObjectsOfType<LLM>())
{
// if (!llm.downloadOnBuild) continue;
// if (llm.modelURL != "") MoveAssetAndMeta(LLMUnitySetup.GetAssetPath(llm.model), Path.Combine(tempDir, Path.GetFileName(llm.model)));
if (llm.loraURL != "") MoveAssetAndMeta(LLMUnitySetup.GetAssetPath(llm.lora), Path.Combine(tempDir, Path.GetFileName(llm.lora)));
}
if (type == LogType.Error) BuildCompleted();
}

static void ResetMoves()
// called after the build
public void OnPostprocessBuild(BuildReport report)
{
if (!File.Exists(movedCache)) return;
List<MovedPair> movedPairs = JsonUtility.FromJson<FoldersMovedWrapper>(File.ReadAllText(movedCache)).movedPairs;
if (movedPairs == null) return;

bool refresh = false;
foreach (var pair in movedPairs) refresh |= MovePath(pair.target, pair.source);
if (refresh) AssetDatabase.Refresh();
File.Delete(movedCache);
BuildCompleted();
}
}

[Serializable]
public struct MovedPair
{
public string source;
public string target;
}

[Serializable]
public class FoldersMovedWrapper
{
public List<MovedPair> movedPairs;
public void BuildCompleted()
{
Application.logMessageReceived -= OnBuildError;
LLMBuilder.Reset();
}
}
}
2 changes: 1 addition & 1 deletion Editor/LLMEditor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public void AddModelLoaders(SerializedObject llmScriptSO, LLM llmScript)
}
EditorGUILayout.EndHorizontal();
}
AddLoadButtons();
_ = AddLoadButtons();
bool downloadOnStart = EditorGUILayout.Toggle("Download on Start", LLMManager.downloadOnStart);
if (downloadOnStart != LLMManager.downloadOnStart)
{
Expand Down
76 changes: 50 additions & 26 deletions Runtime/LLMUnitySetup.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,14 @@ public class LLMUnitySetup
public static string LlamaLibURL = $"https://github.com/undreamai/LlamaLib/releases/download/{LlamaLibVersion}/undreamai-{LlamaLibVersion}-llamacpp.zip";
/// <summary> LlamaLib path </summary>
public static string libraryPath = GetAssetPath(Path.GetFileName(LlamaLibURL).Replace(".zip", ""));
/// <summary> LLMnity store path </summary>
public static string LLMUnityStore = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "LLMUnity");
/// <summary> Model download path </summary>
public static string modelDownloadPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "LLMUnity");
public static string modelDownloadPath = Path.Combine(LLMUnityStore, "models");
/// <summary> Model list for project </summary>
public static string modelListPath = Path.Combine(Application.temporaryCachePath, "modelCache.json");
/// <summary> Temporary dir for build </summary>
public static string buildTempDir = Path.Combine(Application.temporaryCachePath, "LLMUnityBuild");

/// <summary> Default models for download </summary>
[HideInInspector] public static readonly (string, string)[] modelOptions = new(string, string)[]
Expand Down Expand Up @@ -228,11 +232,11 @@ public static async Task AndroidExtractFile(string assetName, bool overwrite = f
string target = GetAssetPath(assetName);
if (!overwrite && File.Exists(target))
{
Debug.Log($"File {target} already exists");
Log($"File {target} already exists");
return;
}

Debug.Log($"Extracting {source} to {target}");
Log($"Extracting {source} to {target}");

// UnityWebRequest to read the file from StreamingAssets
UnityWebRequest www = UnityWebRequest.Get(source);
Expand All @@ -242,7 +246,7 @@ public static async Task AndroidExtractFile(string assetName, bool overwrite = f
while (!operation.isDone) await Task.Delay(1);
if (www.result != UnityWebRequest.Result.Success)
{
Debug.LogError("Failed to load file from StreamingAssets: " + www.error);
LogError("Failed to load file from StreamingAssets: " + www.error);
}
else
{
Expand All @@ -267,6 +271,44 @@ public static bool IsSubPath(string childPath, string parentPath)
}

#if UNITY_EDITOR

public static void CopyPath(string source, string target)
{
if (File.Exists(source))
{
File.Copy(source, target);
}
else if (Directory.Exists(source))
{
Directory.CreateDirectory(target);
List<string> filesAndDirs = new List<string>();
filesAndDirs.AddRange(Directory.GetFiles(source));
filesAndDirs.AddRange(Directory.GetDirectories(source));
foreach (string path in filesAndDirs)
{
CopyPath(path, Path.Combine(target, Path.GetFileName(path)));
}
}
}

public static void MovePath(string source, string target)
{
CopyPath(source, target);
DeletePath(source);
}

public static bool DeletePath(string path)
{
if (!IsSubPath(path, GetAssetPath()) && !IsSubPath(path, buildTempDir))
{
LogError($"Safeguard: {path} will not be deleted because it may not be safe");
return false;
}
if (File.Exists(path)) File.Delete(path);
else if (Directory.Exists(path)) Directory.Delete(path, true);
return true;
}

[HideInInspector] public static float libraryProgress = 1;

private static async Task DownloadLibrary()
Expand All @@ -277,7 +319,9 @@ private static async Task DownloadLibrary()
if (!Directory.Exists(libraryPath))
{
await DownloadFile(LlamaLibURL, libZip, true, null, SetLibraryProgress);
AssetDatabase.StartAssetEditing();
ZipFile.ExtractToDirectory(libZip, libraryPath);
AssetDatabase.StopAssetEditing();
File.Delete(libZip);
}
libraryProgress = 1;
Expand Down Expand Up @@ -307,26 +351,6 @@ public static string AddAsset(string assetPath)
return filename;
}

public static void CreateSymlink(string sourcePath, string targetPath)
{
bool isDirectory = Directory.Exists(sourcePath);
if (!isDirectory && !File.Exists(sourcePath)) throw new FileNotFoundException($"Source path does not exist: {sourcePath}");

bool success;
#if UNITY_STANDALONE_WIN
success = CreateSymbolicLink(targetPath, sourcePath, (int)isDirectory);
#else
success = symlink(sourcePath, targetPath) == 0;
#endif
if (!success) throw new IOException($"Failed to create symbolic link: {targetPath}");
}

[DllImport("kernel32.dll", CharSet = CharSet.Unicode)]
private static extern bool CreateSymbolicLink(string lpSymlinkFileName, string lpTargetFileName, int dwFlags);

[DllImport("libc", SetLastError = true)]
private static extern int symlink(string oldpath, string newpath);

#endif
/// \endcond
public static int GetMaxFreqKHz(int cpuId)
Expand Down Expand Up @@ -422,7 +446,7 @@ public static int AndroidGetNumBigCores()
}
catch (Exception e)
{
Debug.LogError(e.Message);
LogError(e.Message);
}

int numBigCores = 0;
Expand Down Expand Up @@ -474,7 +498,7 @@ public static int AndroidGetNumBigCoresCapacity()
}
catch (Exception e)
{
Debug.LogError(e.Message);
LogError(e.Message);
}

int numBigCores = 0;
Expand Down

0 comments on commit 621ca4c

Please sign in to comment.