Add changes required by Nimble lock file support (nim-lang#12104)

Implemented support for Nimble local cache with package directories with a checksum of the package at the end of their names. Now the compiler supports package paths in the form: * /path_to_nimble_cache_dir/pkgs/package_name-1.2.3- FEBADEAEA2345E777F0F6F8433F7F0A52EDD5D1B * /path_to_nimble_cache_dir/pkgs/package_name-#head- 042D4BE2B90ED0672E717D71850ABDB0A2D19CD2 * /path_to_nimble_cache_dir/pkgs/package_name-#branch-name- DBC1F902CB79946E990E38AF51F0BAD36ACFABD9 Related to nim-lang/nimble#127
PMunch · Mar 28, 2022 · 393d386 · 393d386
1 parent 1828479
commit 393d386
Show file tree

Hide file tree

Showing 5 changed files with 140 additions and 55 deletions.
diff --git a/compiler/modulepaths.nim b/compiler/modulepaths.nim
@@ -21,7 +21,7 @@ when false:
     for k, p in os.walkDir(dir, relative=true):
       if k == pcDir and p.len > pkg.len+1 and
           p[pkg.len] == '-' and p.startsWith(pkg):
-        let (_, a) = getPathVersion(p)
+        let (_, a, _) = getPathVersionChecksum(p)
         if bestv.len == 0 or bestv < a:
           bestv = a
           best = dir / p

diff --git a/compiler/nimblecmd.nim b/compiler/nimblecmd.nim
@@ -9,15 +9,16 @@
 
 ## Implements some helper procs for Nimble (Nim's package manager) support.
 
-import parseutils, strutils, strtabs, os, options, msgs, sequtils,
-  lineinfos, pathutils
+import parseutils, strutils, os, options, msgs, sequtils, lineinfos, pathutils,
+  std/sha1, tables
 
 proc addPath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
   if not conf.searchPaths.contains(path):
     conf.searchPaths.insert(path, 0)
 
 type
   Version* = distinct string
+  PackageInfo = Table[string, tuple[version, checksum: string]]
 
 proc `$`*(ver: Version): string {.borrow.}
 
@@ -62,43 +63,64 @@ proc `<`*(ver: Version, ver2: Version): bool =
     else:
       return false
 
-proc getPathVersion*(p: string): tuple[name, version: string] =
-  ## Splits path ``p`` in the format ``/home/user/.nimble/pkgs/package-0.1``
-  ## into ``(/home/user/.nimble/pkgs/package, 0.1)``
-  result.name = ""
-  result.version = ""
-
-  const specialSeparator = "-#"
-  let last = p.rfind(p.lastPathPart) # the index where the last path part begins
-  var sepIdx = p.find(specialSeparator, start = last)
-  if sepIdx == -1:
-    sepIdx = p.rfind('-', start = last)
-
-  if sepIdx == -1:
-    result.name = p
-    return
+proc getPathVersionChecksum*(p: string): tuple[name, version, checksum: string] =
+  ## Splits path ``p`` in the format
+  ## ``/home/user/.nimble/pkgs/package-0.1-febadeaea2345e777f0f6f8433f7f0a52edd5d1b`` into
+  ## ``("/home/user/.nimble/pkgs/package", "0.1", "febadeaea2345e777f0f6f8433f7f0a52edd5d1b")``
+
+  const checksumSeparator = '-'
+  const versionSeparator = '-'
+  const specialVersionSepartator = "-#"
+  const separatorNotFound = -1
+
+  var checksumSeparatorIndex = p.rfind(checksumSeparator)
+  if checksumSeparatorIndex != separatorNotFound:
+    result.checksum = p.substr(checksumSeparatorIndex + 1)
+    if not result.checksum.isValidSha1Hash():
+      result.checksum = ""
+      checksumSeparatorIndex = p.len()
+  else:
+    checksumSeparatorIndex = p.len()
 
-  for i in sepIdx..<p.len:
-    if p[i] in {DirSep, AltSep}:
-      result.name = p
-      return
+  var versionSeparatorIndex = p.rfind(
+    specialVersionSepartator, 0, checksumSeparatorIndex - 1)
+  if versionSeparatorIndex != separatorNotFound:
+    result.version = p.substr(
+      versionSeparatorIndex + 1, checksumSeparatorIndex - 1)
+  else:
+    versionSeparatorIndex = p.rfind(
+      versionSeparator, 0, checksumSeparatorIndex - 1)
+    if versionSeparatorIndex != separatorNotFound:
+      result.version = p.substr(
+        versionSeparatorIndex + 1, checksumSeparatorIndex - 1)
+    else:
+      versionSeparatorIndex = checksumSeparatorIndex
 
-  result.name = p[0..sepIdx - 1]
-  result.version = p.substr(sepIdx + 1)
+  result.name = p[0..<versionSeparatorIndex]
 
-proc addPackage(conf: ConfigRef; packages: StringTableRef, p: string; info: TLineInfo) =
-  let (name, ver) = getPathVersion(p)
+proc addPackage*(conf: ConfigRef; packages: var PackageInfo, p: string;
+                 info: TLineInfo) =
+  let (name, ver, checksum) = getPathVersionChecksum(p)
   if isValidVersion(ver):
     let version = newVersion(ver)
-    if packages.getOrDefault(name).newVersion < version or
+    if packages.getOrDefault(name).version.newVersion < version or
       (not packages.hasKey(name)):
-      packages[name] = $version
+      if checksum.isValidSha1Hash():
+        packages[name] = ($version, checksum)
+      else:
+        packages[name] = ($version, "")
   else:
     localError(conf, info, "invalid package name: " & p)
 
-iterator chosen(packages: StringTableRef): string =
+iterator chosen(packages: PackageInfo): string =
   for key, val in pairs(packages):
-    let res = if val.len == 0: key else: key & '-' & val
+    var res = key
+    if val.version.len != 0:
+      res &= '-'
+      res &= val.version
+    if val.checksum.len != 0:
+      res &= '-'
+      res &= val.checksum
     yield res
 
 proc addNimblePath(conf: ConfigRef; p: string, info: TLineInfo) =
@@ -118,7 +140,7 @@ proc addNimblePath(conf: ConfigRef; p: string, info: TLineInfo) =
     conf.lazyPaths.insert(AbsoluteDir path, 0)
 
 proc addPathRec(conf: ConfigRef; dir: string, info: TLineInfo) =
-  var packages = newStringTable(modeStyleInsensitive)
+  var packages: PackageInfo
   var pos = dir.len-1
   if dir[pos] in {DirSep, AltSep}: inc(pos)
   for k,p in os.walkDir(dir):

diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim
@@ -275,3 +275,7 @@ proc `==`*(a, b: SecureHash): bool =
 
   # Not a constant-time comparison, but that's acceptable in this context
   Sha1Digest(a) == Sha1Digest(b)
+
+proc isValidSha1Hash*(s: string): bool =
+  ## Checks if a string is a valid sha1 hash sum.
+  s.len == 40 and allCharsInSet(s, HexDigits)
diff --git a/tests/compiler/tnimblecmd.nim b/tests/compiler/tnimblecmd.nim
@@ -1,26 +1,75 @@
-include compiler/[nimblecmd]
+include compiler/[nimblecmd], sets
 
 proc v(s: string): Version = s.newVersion
-# #head is special in the sense that it's assumed to always be newest.
-doAssert v"1.0" < v"#head"
-doAssert v"1.0" < v"1.1"
-doAssert v"1.0.1" < v"1.1"
-doAssert v"1" < v"1.1"
-doAssert v"#aaaqwe" < v"1.1" # We cannot assume that a branch is newer.
-doAssert v"#a111" < v"#head"
-
-let conf = newConfigRef()
-var rr = newStringTable()
-addPackage conf, rr, "irc-#a111", unknownLineInfo
-addPackage conf, rr, "irc-#head", unknownLineInfo
-addPackage conf, rr, "irc-0.1.0", unknownLineInfo
-#addPackage conf, rr, "irc", unknownLineInfo
-#addPackage conf, rr, "another", unknownLineInfo
-addPackage conf, rr, "another-0.1", unknownLineInfo
-
-addPackage conf, rr, "ab-0.1.3", unknownLineInfo
-addPackage conf, rr, "ab-0.1", unknownLineInfo
-addPackage conf, rr, "justone-1.0", unknownLineInfo
-
-doAssert toSeq(rr.chosen) ==
-  @["irc-#head", "ab-0.1.3", "justone-1.0", "another-0.1"]
+
+proc testVersionsComparison =
+  # #head is special in the sense that it's assumed to always be newest.
+  doAssert v"1.0" < v"#head"
+  doAssert v"1.0" < v"1.1"
+  doAssert v"1.0.1" < v"1.1"
+  doAssert v"1" < v"1.1"
+  doAssert v"#aaaqwe" < v"1.1" # We cannot assume that a branch is newer.
+  doAssert v"#a111" < v"#head"
+
+proc testAddPackageWithoutChecksum =
+  ## For backward compatibility it is not required all packages to have a
+  ## sha1 checksum at the end of the name of the Nimble cache directory.
+  ## This way a new compiler will be able to work with an older Nimble.
+
+  let conf = newConfigRef()
+  var rr: PackageInfo
+
+  addPackage conf, rr, "irc-#a111", unknownLineInfo
+  addPackage conf, rr, "irc-#head", unknownLineInfo
+  addPackage conf, rr, "irc-0.1.0", unknownLineInfo
+
+  addPackage conf, rr, "another-0.1", unknownLineInfo
+
+  addPackage conf, rr, "ab-0.1.3", unknownLineInfo
+  addPackage conf, rr, "ab-0.1", unknownLineInfo
+  addPackage conf, rr, "justone-1.0", unknownLineInfo
+
+  doAssert toSeq(rr.chosen).toHashSet ==
+    ["irc-#head", "another-0.1", "ab-0.1.3", "justone-1.0"].toHashSet
+
+proc testAddPackageWithChecksum =
+  let conf = newConfigRef()
+  var rr: PackageInfo
+
+  # in the case of packages with the same version, but different checksums for
+  # now the first one will be chosen
+
+  addPackage conf, rr, "irc-#a111-DBC1F902CB79946E990E38AF51F0BAD36ACFABD9",
+             unknownLineInfo
+  addPackage conf, rr, "irc-#head-042D4BE2B90ED0672E717D71850ABDB0A2D19CD1",
+             unknownLineInfo
+  addPackage conf, rr, "irc-#head-042D4BE2B90ED0672E717D71850ABDB0A2D19CD2",
+             unknownLineInfo
+  addPackage conf, rr, "irc-0.1.0-6EE6DE936B32E82C7DBE526DA3463574F6568FAF",
+             unknownLineInfo
+
+  addPackage conf, rr, "another-0.1", unknownLineInfo
+  addPackage conf, rr, "another-0.1-F07EE6040579F0590608A8FD34F5F2D91D859340",
+             unknownLineInfo
+
+  addPackage conf, rr, "ab-0.1.3-34BC3B72CE46CF5A496D1121CFEA7369385E9EA2",
+             unknownLineInfo
+  addPackage conf, rr, "ab-0.1.3-24BC3B72CE46CF5A496D1121CFEA7369385E9EA2",
+             unknownLineInfo
+  addPackage conf, rr, "ab-0.1-A3CFFABDC4759F7779D541F5E031AED17169390A",
+             unknownLineInfo
+
+  # lower case hex digits is also a valid sha1 checksum
+  addPackage conf, rr, "justone-1.0-f07ee6040579f0590608a8fd34f5f2d91d859340",
+             unknownLineInfo
+
+  doAssert toSeq(rr.chosen).toHashSet == [
+    "irc-#head-042D4BE2B90ED0672E717D71850ABDB0A2D19CD1",
+    "another-0.1",
+    "ab-0.1.3-34BC3B72CE46CF5A496D1121CFEA7369385E9EA2",
+    "justone-1.0-f07ee6040579f0590608a8fd34f5f2d91d859340"
+    ].toHashSet
+
+testVersionsComparison()
+testAddPackageWithoutChecksum()
+testAddPackageWithChecksum()
diff --git a/tests/stdlib/tsha1.nim b/tests/stdlib/tsha1.nim
@@ -11,3 +11,13 @@ checkVector("", "da39a3ee5e6b4b0d3255bfef95601890afd80709")
 checkVector("abc", "a9993e364706816aba3e25717850c26c9cd0d89d")
 checkVector("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
             "84983e441c3bd26ebaae4aa1f95129e5e54670f1")
+
+proc testIsValidSha1Hash =
+  doAssert not isValidSha1Hash("")
+  doAssert not isValidSha1Hash("042D4BE2B90ED0672E717D71850ABDB0A2D19CD11")
+  doAssert not isValidSha1hash("042G4BE2B90ED0672E717D71850ABDB0A2D19CD1")
+  doAssert isValidSha1Hash("042D4BE2B90ED0672E717D71850ABDB0A2D19CD1")
+  doAssert isValidSha1Hash("042d4be2b90ed0672e717d71850abdb0a2d19cd1")
+  doAssert isValidSha1Hash("042d4be2b90ed0672e717D71850ABDB0A2D19CD1")
+
+testIsValidSha1Hash()