Skip to content

Commit

Permalink
improve build process
Browse files Browse the repository at this point in the history
  • Loading branch information
amakropoulos committed Jul 26, 2024
1 parent ee7c6ce commit a1a0b71
Show file tree
Hide file tree
Showing 5 changed files with 209 additions and 148 deletions.
142 changes: 25 additions & 117 deletions Editor/LLMBuildProcessor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,150 +2,58 @@
using UnityEditor.Build;
using UnityEditor.Build.Reporting;
using UnityEngine;
using System.IO;
using System.Collections.Generic;
using System;

namespace LLMUnity
{
public class LLMBuildProcessor : MonoBehaviour, IPreprocessBuildWithReport, IPostprocessBuildWithReport
public class LLMBuildProcessor : IPreprocessBuildWithReport, IPostprocessBuildWithReport
{
public int callbackOrder => 0;
static string tempDir = Path.Combine(Application.temporaryCachePath, "LLMBuildProcessor", Path.GetFileName(LLMUnitySetup.libraryPath));
static List<MovedPair> movedPairs = new List<MovedPair>();
static string movedCache = Path.Combine(tempDir, "moved.json");

[InitializeOnLoadMethod]
private static void InitializeOnLoad()
{
if (!Directory.Exists(tempDir)) Directory.CreateDirectory(tempDir);
else ResetMoves();
}

// CALLED BEFORE THE BUILD
// called before the build
public void OnPreprocessBuild(BuildReport report)
{
// Start listening for errors when build starts
Application.logMessageReceived += OnBuildError;
HideLibraryPlatforms(report.summary.platform);
HideModels();
if (movedPairs.Count > 0) AssetDatabase.Refresh();
}

// CALLED DURING BUILD TO CHECK FOR ERRORS
private void OnBuildError(string condition, string stacktrace, LogType type)
{
if (type == LogType.Error)
{
// FAILED TO BUILD, STOP LISTENING FOR ERRORS
BuildCompleted();
}
}

// CALLED AFTER THE BUILD
public void OnPostprocessBuild(BuildReport report)
{
BuildCompleted();
}

public void BuildCompleted()
{
Application.logMessageReceived -= OnBuildError;
ResetMoves();
}

static bool MovePath(string source, string target)
{
bool moved = false;
if (File.Exists(source))
{
File.Move(source, target);
moved = true;
}
else if (Directory.Exists(source))
{
Directory.Move(source, target);
moved = true;
}
if (moved)
{
movedPairs.Add(new MovedPair {source = source, target = target});
File.WriteAllText(movedCache, JsonUtility.ToJson(new FoldersMovedWrapper { movedPairs = movedPairs }));
}
return moved;
}

static void MoveAssetAndMeta(string source, string target)
{
MovePath(source + ".meta", target + ".meta");
MovePath(source, target);
}

static void HideLibraryPlatforms(BuildTarget buildPlatform)
{
List<string> platforms = new List<string>(){ "windows", "macos", "linux", "android" };
switch (buildPlatform)
string platform = null;
switch (report.summary.platform)
{
case BuildTarget.StandaloneWindows:
case BuildTarget.StandaloneWindows64:
platforms.Remove("windows");
platform = "windows";
break;
case BuildTarget.StandaloneLinux64:
platforms.Remove("linux");
platform = "linux";
break;
case BuildTarget.StandaloneOSX:
platforms.Remove("macos");
platform = "macos";
break;
case BuildTarget.Android:
platforms.Remove("android");
platform = "android";
break;
case BuildTarget.iOS:
platform = "ios";
break;
}

foreach (string dirname in Directory.GetDirectories(LLMUnitySetup.libraryPath))
{
foreach (string platform in platforms)
{
if (Path.GetFileName(dirname).StartsWith(platform))
{
MoveAssetAndMeta(dirname, Path.Combine(tempDir, Path.GetFileName(dirname)));
}
}
}
LLMBuilder.HideLibraryPlatforms(platform);
LLMBuilder.CopyModels();
AssetDatabase.Refresh();
}

static void HideModels()
// called during build to check for errors
private void OnBuildError(string condition, string stacktrace, LogType type)
{
foreach (LLM llm in FindObjectsOfType<LLM>())
{
// if (!llm.downloadOnBuild) continue;
// if (llm.modelURL != "") MoveAssetAndMeta(LLMUnitySetup.GetAssetPath(llm.model), Path.Combine(tempDir, Path.GetFileName(llm.model)));
if (llm.loraURL != "") MoveAssetAndMeta(LLMUnitySetup.GetAssetPath(llm.lora), Path.Combine(tempDir, Path.GetFileName(llm.lora)));
}
if (type == LogType.Error) BuildCompleted();
}

static void ResetMoves()
// called after the build
public void OnPostprocessBuild(BuildReport report)
{
if (!File.Exists(movedCache)) return;
List<MovedPair> movedPairs = JsonUtility.FromJson<FoldersMovedWrapper>(File.ReadAllText(movedCache)).movedPairs;
if (movedPairs == null) return;

bool refresh = false;
foreach (var pair in movedPairs) refresh |= MovePath(pair.target, pair.source);
if (refresh) AssetDatabase.Refresh();
File.Delete(movedCache);
BuildCompleted();
}
}

[Serializable]
public struct MovedPair
{
public string source;
public string target;
}

[Serializable]
public class FoldersMovedWrapper
{
public List<MovedPair> movedPairs;
public void BuildCompleted()
{
Application.logMessageReceived -= OnBuildError;
LLMBuilder.Reset();
}
}
}
6 changes: 3 additions & 3 deletions Editor/LLMEditor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public void AddModelLoaders(SerializedObject llmScriptSO, LLM llmScript)
}
EditorGUILayout.EndHorizontal();
}
AddLoadButtons();
_ = AddLoadButtons();
bool downloadOnStart = EditorGUILayout.Toggle("Download on Start", LLMManager.downloadOnStart);
if (downloadOnStart != LLMManager.downloadOnStart)
{
Expand Down Expand Up @@ -92,8 +92,8 @@ static void ResetModelOptions()
{
List<string> existingOptions = new List<string>();
foreach (ModelEntry entry in LLMManager.modelEntries) existingOptions.Add(entry.url);
modelOptions = new List<string>();
modelURLs = new List<string>();
modelOptions = new List<string>(){"Download model", "Custom URL"};
modelURLs = new List<string>(){null, null};
foreach ((string name, string url) in LLMUnitySetup.modelOptions)
{
if (url != null && existingOptions.Contains(url)) continue;
Expand Down
120 changes: 120 additions & 0 deletions Runtime/LLMBuilder.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
using UnityEditor;
using UnityEngine;
using System.IO;
using System.Collections.Generic;
using System;

#if UNITY_EDITOR
namespace LLMUnity
{
public class LLMBuilder
{
static List<MovedPair> movedPairs = new List<MovedPair>();
static string movedCache = Path.Combine(LLMUnitySetup.buildTempDir, "moved.json");

[InitializeOnLoadMethod]
private static void InitializeOnLoad()
{
Directory.CreateDirectory(LLMUnitySetup.buildTempDir);
Reset();
}

public delegate void ActionCallback(string source, string target);

static void AddMovedPair(string source, string target)
{
movedPairs.Add(new MovedPair {source = source, target = target});
File.WriteAllText(movedCache, JsonUtility.ToJson(new FoldersMovedWrapper { movedPairs = movedPairs }, true));
}

static bool MoveAction(string source, string target, bool addEntry = true)
{
ActionCallback moveCallback;
if (File.Exists(source)) moveCallback = File.Move;
else if (Directory.Exists(source)) moveCallback = LLMUnitySetup.MovePath;
else return false;

if (addEntry) AddMovedPair(source, target);
moveCallback(source, target);
return true;
}

static bool CopyAction(string source, string target, bool addEntry = true)
{
ActionCallback copyCallback;
if (File.Exists(source)) copyCallback = File.Copy;
else if (Directory.Exists(source)) copyCallback = LLMUnitySetup.CopyPath;
else return false;

if (addEntry) AddMovedPair("", target);
copyCallback(source, target);
return true;
}

static bool DeleteAction(string source)
{
return LLMUnitySetup.DeletePath(source);
}

public static void HideLibraryPlatforms(string platform)
{
List<string> platforms = new List<string>(){ "windows", "macos", "linux", "android", "ios" };
platforms.Remove(platform);
foreach (string source in Directory.GetDirectories(LLMUnitySetup.libraryPath))
{
foreach (string platformPrefix in platforms)
{
if (Path.GetFileName(source).StartsWith(platformPrefix))
{
string target = Path.Combine(LLMUnitySetup.buildTempDir, Path.GetFileName(source));
MoveAction(source, target);
MoveAction(source + ".meta", target + ".meta");
}
}
}
}

public static void CopyModels()
{
if (LLMManager.downloadOnStart) return;
foreach (ModelEntry modelEntry in LLMManager.modelEntries)
{
string source = modelEntry.path;
string target = LLMUnitySetup.GetAssetPath(modelEntry.filename);
if (!modelEntry.includeInBuild || File.Exists(target)) continue;
CopyAction(source, target);
AddMovedPair("", target + ".meta");
}
}

public static void Reset()
{
if (!File.Exists(movedCache)) return;
List<MovedPair> movedPairs = JsonUtility.FromJson<FoldersMovedWrapper>(File.ReadAllText(movedCache)).movedPairs;
if (movedPairs == null) return;

bool refresh = false;
foreach (var pair in movedPairs)
{
if (pair.source == "") refresh |= DeleteAction(pair.target);
else refresh |= MoveAction(pair.target, pair.source, false);
}
if (refresh) AssetDatabase.Refresh();
LLMUnitySetup.DeletePath(movedCache);
}
}

[Serializable]
public struct MovedPair
{
public string source;
public string target;
}

[Serializable]
public class FoldersMovedWrapper
{
public List<MovedPair> movedPairs;
}
}
#endif
11 changes: 11 additions & 0 deletions Runtime/LLMBuilder.cs.meta

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit a1a0b71

Please sign in to comment.