From 621ca4cc0c882794a304954a59e0b8606822a49f Mon Sep 17 00:00:00 2001 From: Antonis Makropoulos Date: Fri, 26 Jul 2024 13:15:09 +0300 Subject: [PATCH] improve build process --- Editor/LLMBuildProcessor.cs | 142 +++++++----------------------------- Editor/LLMEditor.cs | 2 +- Runtime/LLMUnitySetup.cs | 76 ++++++++++++------- 3 files changed, 76 insertions(+), 144 deletions(-) diff --git a/Editor/LLMBuildProcessor.cs b/Editor/LLMBuildProcessor.cs index 05cc6238..b98caf0f 100644 --- a/Editor/LLMBuildProcessor.cs +++ b/Editor/LLMBuildProcessor.cs @@ -2,150 +2,58 @@ using UnityEditor.Build; using UnityEditor.Build.Reporting; using UnityEngine; -using System.IO; -using System.Collections.Generic; -using System; namespace LLMUnity { - public class LLMBuildProcessor : MonoBehaviour, IPreprocessBuildWithReport, IPostprocessBuildWithReport + public class LLMBuildProcessor : IPreprocessBuildWithReport, IPostprocessBuildWithReport { public int callbackOrder => 0; - static string tempDir = Path.Combine(Application.temporaryCachePath, "LLMBuildProcessor", Path.GetFileName(LLMUnitySetup.libraryPath)); - static List movedPairs = new List(); - static string movedCache = Path.Combine(tempDir, "moved.json"); - [InitializeOnLoadMethod] - private static void InitializeOnLoad() - { - if (!Directory.Exists(tempDir)) Directory.CreateDirectory(tempDir); - else ResetMoves(); - } - - // CALLED BEFORE THE BUILD + // called before the build public void OnPreprocessBuild(BuildReport report) { - // Start listening for errors when build starts Application.logMessageReceived += OnBuildError; - HideLibraryPlatforms(report.summary.platform); - HideModels(); - if (movedPairs.Count > 0) AssetDatabase.Refresh(); - } - - // CALLED DURING BUILD TO CHECK FOR ERRORS - private void OnBuildError(string condition, string stacktrace, LogType type) - { - if (type == LogType.Error) - { - // FAILED TO BUILD, STOP LISTENING FOR ERRORS - BuildCompleted(); - } - } - - // CALLED AFTER THE BUILD - public void OnPostprocessBuild(BuildReport report) - { - BuildCompleted(); - } - - public void BuildCompleted() - { - Application.logMessageReceived -= OnBuildError; - ResetMoves(); - } - - static bool MovePath(string source, string target) - { - bool moved = false; - if (File.Exists(source)) - { - File.Move(source, target); - moved = true; - } - else if (Directory.Exists(source)) - { - Directory.Move(source, target); - moved = true; - } - if (moved) - { - movedPairs.Add(new MovedPair {source = source, target = target}); - File.WriteAllText(movedCache, JsonUtility.ToJson(new FoldersMovedWrapper { movedPairs = movedPairs })); - } - return moved; - } - - static void MoveAssetAndMeta(string source, string target) - { - MovePath(source + ".meta", target + ".meta"); - MovePath(source, target); - } - - static void HideLibraryPlatforms(BuildTarget buildPlatform) - { - List platforms = new List(){ "windows", "macos", "linux", "android" }; - switch (buildPlatform) + string platform = null; + switch (report.summary.platform) { case BuildTarget.StandaloneWindows: case BuildTarget.StandaloneWindows64: - platforms.Remove("windows"); + platform = "windows"; break; case BuildTarget.StandaloneLinux64: - platforms.Remove("linux"); + platform = "linux"; break; case BuildTarget.StandaloneOSX: - platforms.Remove("macos"); + platform = "macos"; break; case BuildTarget.Android: - platforms.Remove("android"); + platform = "android"; + break; + case BuildTarget.iOS: + platform = "ios"; break; } - - foreach (string dirname in Directory.GetDirectories(LLMUnitySetup.libraryPath)) - { - foreach (string platform in platforms) - { - if (Path.GetFileName(dirname).StartsWith(platform)) - { - MoveAssetAndMeta(dirname, Path.Combine(tempDir, Path.GetFileName(dirname))); - } - } - } + LLMBuilder.HideLibraryPlatforms(platform); + LLMBuilder.CopyModels(); + AssetDatabase.Refresh(); } - static void HideModels() + // called during build to check for errors + private void OnBuildError(string condition, string stacktrace, LogType type) { - foreach (LLM llm in FindObjectsOfType()) - { - // if (!llm.downloadOnBuild) continue; - // if (llm.modelURL != "") MoveAssetAndMeta(LLMUnitySetup.GetAssetPath(llm.model), Path.Combine(tempDir, Path.GetFileName(llm.model))); - if (llm.loraURL != "") MoveAssetAndMeta(LLMUnitySetup.GetAssetPath(llm.lora), Path.Combine(tempDir, Path.GetFileName(llm.lora))); - } + if (type == LogType.Error) BuildCompleted(); } - static void ResetMoves() + // called after the build + public void OnPostprocessBuild(BuildReport report) { - if (!File.Exists(movedCache)) return; - List movedPairs = JsonUtility.FromJson(File.ReadAllText(movedCache)).movedPairs; - if (movedPairs == null) return; - - bool refresh = false; - foreach (var pair in movedPairs) refresh |= MovePath(pair.target, pair.source); - if (refresh) AssetDatabase.Refresh(); - File.Delete(movedCache); + BuildCompleted(); } - } - [Serializable] - public struct MovedPair - { - public string source; - public string target; - } - - [Serializable] - public class FoldersMovedWrapper - { - public List movedPairs; + public void BuildCompleted() + { + Application.logMessageReceived -= OnBuildError; + LLMBuilder.Reset(); + } } } diff --git a/Editor/LLMEditor.cs b/Editor/LLMEditor.cs index c78cef3f..4dbab6d9 100644 --- a/Editor/LLMEditor.cs +++ b/Editor/LLMEditor.cs @@ -63,7 +63,7 @@ public void AddModelLoaders(SerializedObject llmScriptSO, LLM llmScript) } EditorGUILayout.EndHorizontal(); } - AddLoadButtons(); + _ = AddLoadButtons(); bool downloadOnStart = EditorGUILayout.Toggle("Download on Start", LLMManager.downloadOnStart); if (downloadOnStart != LLMManager.downloadOnStart) { diff --git a/Runtime/LLMUnitySetup.cs b/Runtime/LLMUnitySetup.cs index abe96d72..3674abb4 100644 --- a/Runtime/LLMUnitySetup.cs +++ b/Runtime/LLMUnitySetup.cs @@ -78,10 +78,14 @@ public class LLMUnitySetup public static string LlamaLibURL = $"https://github.com/undreamai/LlamaLib/releases/download/{LlamaLibVersion}/undreamai-{LlamaLibVersion}-llamacpp.zip"; /// LlamaLib path public static string libraryPath = GetAssetPath(Path.GetFileName(LlamaLibURL).Replace(".zip", "")); + /// LLMnity store path + public static string LLMUnityStore = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "LLMUnity"); /// Model download path - public static string modelDownloadPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "LLMUnity"); + public static string modelDownloadPath = Path.Combine(LLMUnityStore, "models"); /// Model list for project public static string modelListPath = Path.Combine(Application.temporaryCachePath, "modelCache.json"); + /// Temporary dir for build + public static string buildTempDir = Path.Combine(Application.temporaryCachePath, "LLMUnityBuild"); /// Default models for download [HideInInspector] public static readonly (string, string)[] modelOptions = new(string, string)[] @@ -228,11 +232,11 @@ public static async Task AndroidExtractFile(string assetName, bool overwrite = f string target = GetAssetPath(assetName); if (!overwrite && File.Exists(target)) { - Debug.Log($"File {target} already exists"); + Log($"File {target} already exists"); return; } - Debug.Log($"Extracting {source} to {target}"); + Log($"Extracting {source} to {target}"); // UnityWebRequest to read the file from StreamingAssets UnityWebRequest www = UnityWebRequest.Get(source); @@ -242,7 +246,7 @@ public static async Task AndroidExtractFile(string assetName, bool overwrite = f while (!operation.isDone) await Task.Delay(1); if (www.result != UnityWebRequest.Result.Success) { - Debug.LogError("Failed to load file from StreamingAssets: " + www.error); + LogError("Failed to load file from StreamingAssets: " + www.error); } else { @@ -267,6 +271,44 @@ public static bool IsSubPath(string childPath, string parentPath) } #if UNITY_EDITOR + + public static void CopyPath(string source, string target) + { + if (File.Exists(source)) + { + File.Copy(source, target); + } + else if (Directory.Exists(source)) + { + Directory.CreateDirectory(target); + List filesAndDirs = new List(); + filesAndDirs.AddRange(Directory.GetFiles(source)); + filesAndDirs.AddRange(Directory.GetDirectories(source)); + foreach (string path in filesAndDirs) + { + CopyPath(path, Path.Combine(target, Path.GetFileName(path))); + } + } + } + + public static void MovePath(string source, string target) + { + CopyPath(source, target); + DeletePath(source); + } + + public static bool DeletePath(string path) + { + if (!IsSubPath(path, GetAssetPath()) && !IsSubPath(path, buildTempDir)) + { + LogError($"Safeguard: {path} will not be deleted because it may not be safe"); + return false; + } + if (File.Exists(path)) File.Delete(path); + else if (Directory.Exists(path)) Directory.Delete(path, true); + return true; + } + [HideInInspector] public static float libraryProgress = 1; private static async Task DownloadLibrary() @@ -277,7 +319,9 @@ private static async Task DownloadLibrary() if (!Directory.Exists(libraryPath)) { await DownloadFile(LlamaLibURL, libZip, true, null, SetLibraryProgress); + AssetDatabase.StartAssetEditing(); ZipFile.ExtractToDirectory(libZip, libraryPath); + AssetDatabase.StopAssetEditing(); File.Delete(libZip); } libraryProgress = 1; @@ -307,26 +351,6 @@ public static string AddAsset(string assetPath) return filename; } - public static void CreateSymlink(string sourcePath, string targetPath) - { - bool isDirectory = Directory.Exists(sourcePath); - if (!isDirectory && !File.Exists(sourcePath)) throw new FileNotFoundException($"Source path does not exist: {sourcePath}"); - - bool success; -#if UNITY_STANDALONE_WIN - success = CreateSymbolicLink(targetPath, sourcePath, (int)isDirectory); -#else - success = symlink(sourcePath, targetPath) == 0; -#endif - if (!success) throw new IOException($"Failed to create symbolic link: {targetPath}"); - } - - [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] - private static extern bool CreateSymbolicLink(string lpSymlinkFileName, string lpTargetFileName, int dwFlags); - - [DllImport("libc", SetLastError = true)] - private static extern int symlink(string oldpath, string newpath); - #endif /// \endcond public static int GetMaxFreqKHz(int cpuId) @@ -422,7 +446,7 @@ public static int AndroidGetNumBigCores() } catch (Exception e) { - Debug.LogError(e.Message); + LogError(e.Message); } int numBigCores = 0; @@ -474,7 +498,7 @@ public static int AndroidGetNumBigCoresCapacity() } catch (Exception e) { - Debug.LogError(e.Message); + LogError(e.Message); } int numBigCores = 0;