deno per package cache derivations + hashes

flake.nix

@@ -10,16 +10,24 @@
     flake-utils.lib.eachDefaultSystem (system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
+        lib = {
+          collectDenoCaches = import ./nix/deno/collect_caches.nix;
+          readDenoScroll = import ./nix/deno/read_scroll.nix;
+        };
         shelf = import ./shelf {
-          inherit pkgs system;
+          inherit pkgs system lib;
         };
       in
         {
-          packages.shelf = shelf;
+          inherit lib;
+
           devShells.default = pkgs.mkShell {
-            buildInputs = [  ];
+            buildInputs = [ pkgs.deno ];
             shellHook = ''
               ${shelf.setupScript}
+
+              echo "Availabe scrolls:"
+              echo ${pkgs.lib.escapeShellArg (builtins.toJSON shelf.scrolls)} | jq .
             '';
           };
         });

nix/deno/collect_caches.nix

@@ -0,0 +1,42 @@
+{ pkgs, denoCacheDrvs }:
+let
+  mergeOneCache = cache: ''
+    if [ -d "${cache}" ]; then
+      echo "  Merging cache from ${cache}"
+      # Use cp with fallback, handle overlapping files gracefully
+      cp -rL "${cache}"/* "$out/" 2>/dev/null || true
+    else
+      echo "  Warning: ${cache} is not a directory, skipping"
+    fi
+  '';
+
+  mergeCacheCmd = pkgs.runCommand "deno-shelf-shared-cache"
+    {
+      buildInputs = denoCacheDrvs;
+    } ''
+    mkdir -p $out
+
+    echo "Merging ${builtins.toString (builtins.length denoCacheDrvs)} Deno caches..."
+
+    ${pkgs.lib.concatMapStringsSep "\n" mergeOneCache denoCacheDrvs}
+
+    echo "Deno caches merged. Final size: $(du -sh $out | cut -f1)"
+  '';
+in
+
+pkgs.writeShellScript "deno-shelf-setup" ''
+  export DENO_DIR=$PWD/.deno_cache
+
+  if [ ! -d "$DENO_DIR" ] || [ "${mergeCacheCmd}" -nt "$DENO_DIR/.cache_timestamp" ]; then
+    echo "Setting up mutable Deno cache..."
+
+    rm -rf "$DENO_DIR"
+    cp -r "${mergeCacheCmd}" "$DENO_DIR"
+    chmod -R u+w "$DENO_DIR"
+
+    touch "$DENO_DIR/.cache_timestamp"
+    echo "Deno cache initialized"
+  else
+    echo "Deno cache already up to date"
+  fi
+''

nix/deno/read_scroll.nix

@@ -0,0 +1,64 @@
+{ pkgs, system, scrollsDir, subDir }:
+let
+  target =
+    let
+      arch = builtins.head (builtins.split "-" system);
+      os = builtins.elemAt (builtins.split "-" system) 1;
+      vendor = if os == "darwin" then "apple" else "unknown";
+      sys = if os == "darwin" then "darwin" else "linux-gnu";
+    in "${arch}-${vendor}-${sys}";
+
+  scroll = import "${scrollsDir}/${subDir}/scroll.nix";
+
+  hashFilesCat = builtins.concatStringsSep ""
+    (builtins.map builtins.readFile scroll.build.hashFiles);
+  sourceContentHash_ = builtins.convertHash {
+    hash = "sha1:${builtins.hashString "sha1" hashFilesCat}";
+    toHashFormat = "base64";
+  };
+  sourceContentHash = "sha1-${sourceContentHash_}";
+  outputHash =
+    scroll.build.knownHashes.${sourceContentHash}
+      or pkgs.lib.fakeSha256;
+in
+{
+  env = pkgs.stdenv.mkDerivation {
+    name = "${scroll.name}-scroll-env";
+    src = scrollsDir;
+
+    nativeBuildInputs = [ pkgs.deno pkgs.jq ];
+    dontPatchShebangs = true;
+
+    buildPhase = ''
+      export DENO_DIR=$out
+
+      cd ${subDir}
+      ${scroll.build.cacheCommand}
+      cd ..
+
+      echo 'Go fuck yourself, SQLite!'
+      find $out -name "*-wal" -delete
+      find $out -name "*-shm" -delete
+
+      echo 'Go fuck yourself, JSON!'
+      find $out/npm -name "*.json" -type f 2>/dev/null | while read -r file; do
+        if ${pkgs.jq}/bin/jq empty "$file" 2>/dev/null; then
+          ${pkgs.jq}/bin/jq -S . "$file" > "$file.tmp" && mv "$file.tmp" "$file"
+        fi
+      done
+
+      echo 'sourceContentHash: ${sourceContentHash}'
+      echo ""
+      echo "add me to knownHashes if you see a fake sha256 error ^"
+      echo "DON'T USE JSR DEPS UNLESS YOU MAKE THEM REPRODUCIBLE SOMEHOW"
+      echo ""
+    '';
+
+    installPhase = "true";
+
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+    inherit outputHash;
+  };
+  meta = builtins.removeAttrs scroll [ "build" ];
+}

shelf/default.nix

@@ -1,24 +1,25 @@
-{ pkgs, system }:
-let
-  denoShelf = import ./deno {
-    inherit pkgs system;
-    # collective hash of all (relevant) deno sources
-    # mapped to resulting shared deno cache hash
-    knownDenoHashes = {
-      "2JQI9Ie/fkl5Ltr22BGJynF9tFc=" =
-        "sha256-7csbrTbGd6J4a0Gkj9ndKhNxv90KeOHjr7DDNal3L4Q=";
-    };
-  };
-  
-  shelfSetup = pkgs.writeShellScript "setup-shelf" ''
-    export DENO_DIR=$PWD/.deno_cache
-    rm -rf $DENO_DIR
-    cp -r ${denoShelf.cache} $DENO_DIR
-    chmod -R u+w "$DENO_DIR"
+{ pkgs, system, lib }:
+let  
+  denoScrollsDir = ./deno;
 
-    # TODO: add other interpreters
-  '';
+  mkScroll = subDir: lib.readDenoScroll {
+    inherit pkgs system subDir;
+    scrollsDir = denoScrollsDir;
+  };
+
+  scrolls = builtins.map mkScroll [
+    "hjq"
+    "uses-hjq"
+  ];
+
+  shelfSetup =
+    lib.collectDenoCaches {
+      inherit pkgs;
+      denoCacheDrvs = builtins.map (s: s.env) scrolls;
+    };
 in
 {
   setupScript = shelfSetup;
+
+  scrolls = builtins.map (s: s.meta) scrolls;
 }

shelf/deno/default.nix

@@ -1,103 +0,0 @@
-{ pkgs, system, knownDenoHashes }:
-
-let
-  target =
-    let
-      arch = builtins.head (builtins.split "-" system);
-      os = builtins.elemAt (builtins.split "-" system) 1;
-      vendor = if os == "darwin" then "apple" else "unknown";
-      sys = if os == "darwin" then "darwin" else "linux-gnu";
-    in "${arch}-${vendor}-${sys}";
-
-  # currently unused, needed for deno compile
-  denort = pkgs.fetchzip {
-    url = "https://dl.deno.land/release/v${pkgs.deno.version}/denort-${target}.zip";
-    hash = "sha256-ukIk8K2CE+N+3eFs++RPiGZlhhRRVk1gjhdt77/s+4o=";
-  };
-  
-  scrollsDir = ./.;
-  lib = pkgs.lib;
-  packageDirs = [ "./hjq" "./uses-hjq" ];
-
-  # hash only dependency-relevant files (lock files and configs)
-  hashDependencyFiles = dir:
-    let
-      entries = builtins.readDir dir;
-
-      # Only process directories and dependency-relevant files
-      relevantEntry = name: type:
-        if type == "directory"
-        then hashDependencyFiles (dir + "/${name}")
-        else if (lib.hasSuffix ".lock" name ||
-                 lib.hasSuffix ".json" name ||
-                 lib.hasSuffix ".jsonc" name)
-        then builtins.readFile (dir + "/${name}")
-        else null;
-
-      contentMap = lib.filterAttrs (_: v: v != null)
-        (builtins.mapAttrs relevantEntry entries);
-    in builtins.toJSON contentMap;
-
-  sourceContentHash = builtins.convertHash {
-    hash = "sha1:${builtins.hashString "sha1" (hashDependencyFiles scrollsDir)}";
-    toHashFormat = "base64";
-  };
-
-  # we assume that same lock+config files yield same deno caches here
-  # however, jsons need to be sorted and some sqlite files deleted
-
-  # jsr gentlemen specifically require their package metadata
-  # to have multiple timestamps, so they can fuck off
-  outputHash = knownDenoHashes.${sourceContentHash} or lib.fakeSha256;
-
-  unifiedCache = pkgs.stdenv.mkDerivation {
-    name = "deno-scrolls-cache";
-    src = scrollsDir;
-    
-    nativeBuildInputs = [ pkgs.deno pkgs.jq ];
-    dontPatchShebangs = true;
-
-    buildPhase = ''
-      export DENO_DIR=$out
-      
-      ${lib.concatStringsSep "\n" (map (pkg: 
-        let config = import (scrollsDir + "/${pkg}/scroll.nix");
-        in ''
-          echo "Caching ${pkg}..."
-          cd ${pkg}
-          ${config.cache-command}
-          cd ..
-        ''
-      ) packageDirs)}
-
-      echo 'Go fuck yourself, SQLite!'
-      find $out -name "*-wal" -delete
-      find $out -name "*-shm" -delete
-
-      echo 'Go fuck yourself, JSON!'
-      find $out/npm -name "*.json" -type f 2>/dev/null | while read -r file; do
-        if ${pkgs.jq}/bin/jq empty "$file" 2>/dev/null; then
-          ${pkgs.jq}/bin/jq -S . "$file" > "$file.tmp" && mv "$file.tmp" "$file"
-        fi
-      done
-
-      echo 'sourceContentHash: ${sourceContentHash}'
-      echo ""
-      echo "add me to knownHashes if you see a fake sha256 error ^"
-      echo "DON'T USE JSR DEPS UNLESS YOU MAKE THEM REPRODUCIBLE SOMEHOW"
-      echo ""
-    '';
-    
-    installPhase = "true";
-    
-    outputHashMode = "recursive";
-    outputHashAlgo = "sha256";
-    # this is basically FOD on steroids cause it
-    # deliberately breaks on relevant source changes
-    inherit outputHash;
-  };
-  
-in
-{
-  cache = unifiedCache;
-}

shelf/deno/hjq/deno.lock

@@ -1,15 +1,26 @@
 {
-  "version": "4",
+  "version": "5",
   "specifiers": {
+    "npm:@types/node@*": "22.13.4",
     "npm:hjson@^3.2.2": "3.2.2",
     "npm:jq-web@~0.6.2": "0.6.2"
   },
   "npm": {
+    "@types/node@22.13.4": {
+      "integrity": "sha512-ywP2X0DYtX3y08eFVx5fNIw7/uIv8hYUKgXoK8oayJlLnKcRfEYCxWMVE1XagUdVtCJlZT1AU4LXEABW+L1Peg==",
+      "dependencies": [
+        "undici-types"
+      ]
+    },
     "hjson@3.2.2": {
-      "integrity": "sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q=="
+      "integrity": "sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q==",
+      "bin": true
     },
     "jq-web@0.6.2": {
       "integrity": "sha512-+7XvjBYwTx4vP5PYkf6Q6orubO/v+UgMU6By1GritrmShr9QpT3UKa4ANzXWQfhdqtBnQYXsm7ZNbdIHT6tYpQ=="
+    },
+    "undici-types@6.20.0": {
+      "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="
     }
   },
   "workspace": {

shelf/deno/hjq/scroll.nix

@@ -8,7 +8,7 @@
       format = "HJSON";
       react  = "push";
     };
-    jq_filter = {
+    jqFilter = {
       format = "string";
       react  = "push";
     };
@@ -20,6 +20,13 @@
       react  = "push";
     };
   };
-  
-  cache-command = "deno cache --reload main.ts";
+
+  build = {
+    hashFiles = [ ./deno.json ./deno.lock ];
+    cacheCommand = "deno cache --frozen main.ts";
+    knownHashes = {
+      "sha1-gSeoE0sSj+dFQ7SUoyaQV0X/KJE=" =
+        "sha256-0ZfdWVfShbhgAoSse9vEpFPRh5XTTVjgxGMmezwon9I=";
+    };
+  };
 }

shelf/deno/uses-hjq/deno.lock

@@ -1,15 +1,26 @@
 {
-  "version": "4",
+  "version": "5",
   "specifiers": {
+    "npm:@types/node@*": "22.13.4",
     "npm:hjson@^3.2.2": "3.2.2",
     "npm:jq-web@~0.6.2": "0.6.2"
   },
   "npm": {
+    "@types/node@22.13.4": {
+      "integrity": "sha512-ywP2X0DYtX3y08eFVx5fNIw7/uIv8hYUKgXoK8oayJlLnKcRfEYCxWMVE1XagUdVtCJlZT1AU4LXEABW+L1Peg==",
+      "dependencies": [
+        "undici-types"
+      ]
+    },
     "hjson@3.2.2": {
-      "integrity": "sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q=="
+      "integrity": "sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q==",
+      "bin": true
     },
     "jq-web@0.6.2": {
       "integrity": "sha512-+7XvjBYwTx4vP5PYkf6Q6orubO/v+UgMU6By1GritrmShr9QpT3UKa4ANzXWQfhdqtBnQYXsm7ZNbdIHT6tYpQ=="
+    },
+    "undici-types@6.20.0": {
+      "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="
     }
   }
 }

shelf/deno/uses-hjq/scroll.nix

@@ -1,3 +1,32 @@
 {
-  cache-command = "deno cache -I --reload main.ts";
+  name = "uses-hjq";
+
+  description = "A test scroll that mimicks hjq's behavior";
+
+  inputs = {
+    stdin = {
+      format = "HJSON";
+      react  = "push";
+    };
+    jqFilter = {
+      format = "string";
+      react  = "push";
+    };
+  };
+
+  outputs = {
+    stdout = {
+      format = "HJSON";
+      react  = "push";
+    };
+  };
+
+  build = {
+    hashFiles = [ ./deno.json ./deno.lock ];
+    cacheCommand = "deno cache --frozen main.ts";
+    knownHashes = {
+      "sha1-uO43Rt1F6+Ud1wk2p9LQhfP180M=" =
+        "sha256-x/Fvn/l3J7C0D7AftlbRk7iuc6o9qPakdnHb72TSxec=";
+    } ; 
+  };
 }