From f459ec8b781d396233fc12b1e671700c4a12c426 Mon Sep 17 00:00:00 2001 From: EatThePooh Date: Sat, 2 Aug 2025 14:14:41 +0700 Subject: [PATCH] deno per package cache derivations + hashes --- flake.nix | 14 ++++- nix/deno/collect_caches.nix | 42 ++++++++++++++ nix/deno/read_scroll.nix | 64 ++++++++++++++++++++ shelf/default.nix | 39 +++++++------ shelf/deno/default.nix | 103 --------------------------------- shelf/deno/hjq/deno.lock | 15 ++++- shelf/deno/hjq/scroll.nix | 13 ++++- shelf/deno/uses-hjq/deno.lock | 15 ++++- shelf/deno/uses-hjq/scroll.nix | 31 +++++++++- 9 files changed, 203 insertions(+), 133 deletions(-) create mode 100644 nix/deno/collect_caches.nix create mode 100644 nix/deno/read_scroll.nix delete mode 100644 shelf/deno/default.nix diff --git a/flake.nix b/flake.nix index d65e5cb..cda9162 100644 --- a/flake.nix +++ b/flake.nix @@ -10,16 +10,24 @@ flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; + lib = { + collectDenoCaches = import ./nix/deno/collect_caches.nix; + readDenoScroll = import ./nix/deno/read_scroll.nix; + }; shelf = import ./shelf { - inherit pkgs system; + inherit pkgs system lib; }; in { - packages.shelf = shelf; + inherit lib; + devShells.default = pkgs.mkShell { - buildInputs = [ ]; + buildInputs = [ pkgs.deno ]; shellHook = '' ${shelf.setupScript} + + echo "Availabe scrolls:" + echo ${pkgs.lib.escapeShellArg (builtins.toJSON shelf.scrolls)} | jq . ''; }; }); diff --git a/nix/deno/collect_caches.nix b/nix/deno/collect_caches.nix new file mode 100644 index 0000000..865f28a --- /dev/null +++ b/nix/deno/collect_caches.nix @@ -0,0 +1,42 @@ +{ pkgs, denoCacheDrvs }: +let + mergeOneCache = cache: '' + if [ -d "${cache}" ]; then + echo " Merging cache from ${cache}" + # Use cp with fallback, handle overlapping files gracefully + cp -rL "${cache}"/* "$out/" 2>/dev/null || true + else + echo " Warning: ${cache} is not a directory, skipping" + fi + ''; + + mergeCacheCmd = pkgs.runCommand "deno-shelf-shared-cache" + { + buildInputs = denoCacheDrvs; + } '' + mkdir -p $out + + echo "Merging ${builtins.toString (builtins.length denoCacheDrvs)} Deno caches..." + + ${pkgs.lib.concatMapStringsSep "\n" mergeOneCache denoCacheDrvs} + + echo "Deno caches merged. Final size: $(du -sh $out | cut -f1)" + ''; +in + +pkgs.writeShellScript "deno-shelf-setup" '' + export DENO_DIR=$PWD/.deno_cache + + if [ ! -d "$DENO_DIR" ] || [ "${mergeCacheCmd}" -nt "$DENO_DIR/.cache_timestamp" ]; then + echo "Setting up mutable Deno cache..." + + rm -rf "$DENO_DIR" + cp -r "${mergeCacheCmd}" "$DENO_DIR" + chmod -R u+w "$DENO_DIR" + + touch "$DENO_DIR/.cache_timestamp" + echo "Deno cache initialized" + else + echo "Deno cache already up to date" + fi +'' diff --git a/nix/deno/read_scroll.nix b/nix/deno/read_scroll.nix new file mode 100644 index 0000000..312ecac --- /dev/null +++ b/nix/deno/read_scroll.nix @@ -0,0 +1,64 @@ +{ pkgs, system, scrollsDir, subDir }: +let + target = + let + arch = builtins.head (builtins.split "-" system); + os = builtins.elemAt (builtins.split "-" system) 1; + vendor = if os == "darwin" then "apple" else "unknown"; + sys = if os == "darwin" then "darwin" else "linux-gnu"; + in "${arch}-${vendor}-${sys}"; + + scroll = import "${scrollsDir}/${subDir}/scroll.nix"; + + hashFilesCat = builtins.concatStringsSep "" + (builtins.map builtins.readFile scroll.build.hashFiles); + sourceContentHash_ = builtins.convertHash { + hash = "sha1:${builtins.hashString "sha1" hashFilesCat}"; + toHashFormat = "base64"; + }; + sourceContentHash = "sha1-${sourceContentHash_}"; + outputHash = + scroll.build.knownHashes.${sourceContentHash} + or pkgs.lib.fakeSha256; +in +{ + env = pkgs.stdenv.mkDerivation { + name = "${scroll.name}-scroll-env"; + src = scrollsDir; + + nativeBuildInputs = [ pkgs.deno pkgs.jq ]; + dontPatchShebangs = true; + + buildPhase = '' + export DENO_DIR=$out + + cd ${subDir} + ${scroll.build.cacheCommand} + cd .. + + echo 'Go fuck yourself, SQLite!' + find $out -name "*-wal" -delete + find $out -name "*-shm" -delete + + echo 'Go fuck yourself, JSON!' + find $out/npm -name "*.json" -type f 2>/dev/null | while read -r file; do + if ${pkgs.jq}/bin/jq empty "$file" 2>/dev/null; then + ${pkgs.jq}/bin/jq -S . "$file" > "$file.tmp" && mv "$file.tmp" "$file" + fi + done + + echo 'sourceContentHash: ${sourceContentHash}' + echo "" + echo "add me to knownHashes if you see a fake sha256 error ^" + echo "DON'T USE JSR DEPS UNLESS YOU MAKE THEM REPRODUCIBLE SOMEHOW" + echo "" + ''; + + installPhase = "true"; + + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + inherit outputHash; + }; + meta = builtins.removeAttrs scroll [ "build" ]; +} diff --git a/shelf/default.nix b/shelf/default.nix index efcb4de..3c2a66c 100644 --- a/shelf/default.nix +++ b/shelf/default.nix @@ -1,24 +1,25 @@ -{ pkgs, system }: -let - denoShelf = import ./deno { - inherit pkgs system; - # collective hash of all (relevant) deno sources - # mapped to resulting shared deno cache hash - knownDenoHashes = { - "2JQI9Ie/fkl5Ltr22BGJynF9tFc=" = - "sha256-7csbrTbGd6J4a0Gkj9ndKhNxv90KeOHjr7DDNal3L4Q="; - }; - }; - - shelfSetup = pkgs.writeShellScript "setup-shelf" '' - export DENO_DIR=$PWD/.deno_cache - rm -rf $DENO_DIR - cp -r ${denoShelf.cache} $DENO_DIR - chmod -R u+w "$DENO_DIR" +{ pkgs, system, lib }: +let + denoScrollsDir = ./deno; - # TODO: add other interpreters - ''; + mkScroll = subDir: lib.readDenoScroll { + inherit pkgs system subDir; + scrollsDir = denoScrollsDir; + }; + + scrolls = builtins.map mkScroll [ + "hjq" + "uses-hjq" + ]; + + shelfSetup = + lib.collectDenoCaches { + inherit pkgs; + denoCacheDrvs = builtins.map (s: s.env) scrolls; + }; in { setupScript = shelfSetup; + + scrolls = builtins.map (s: s.meta) scrolls; } diff --git a/shelf/deno/default.nix b/shelf/deno/default.nix deleted file mode 100644 index 45160bd..0000000 --- a/shelf/deno/default.nix +++ /dev/null @@ -1,103 +0,0 @@ -{ pkgs, system, knownDenoHashes }: - -let - target = - let - arch = builtins.head (builtins.split "-" system); - os = builtins.elemAt (builtins.split "-" system) 1; - vendor = if os == "darwin" then "apple" else "unknown"; - sys = if os == "darwin" then "darwin" else "linux-gnu"; - in "${arch}-${vendor}-${sys}"; - - # currently unused, needed for deno compile - denort = pkgs.fetchzip { - url = "https://dl.deno.land/release/v${pkgs.deno.version}/denort-${target}.zip"; - hash = "sha256-ukIk8K2CE+N+3eFs++RPiGZlhhRRVk1gjhdt77/s+4o="; - }; - - scrollsDir = ./.; - lib = pkgs.lib; - packageDirs = [ "./hjq" "./uses-hjq" ]; - - # hash only dependency-relevant files (lock files and configs) - hashDependencyFiles = dir: - let - entries = builtins.readDir dir; - - # Only process directories and dependency-relevant files - relevantEntry = name: type: - if type == "directory" - then hashDependencyFiles (dir + "/${name}") - else if (lib.hasSuffix ".lock" name || - lib.hasSuffix ".json" name || - lib.hasSuffix ".jsonc" name) - then builtins.readFile (dir + "/${name}") - else null; - - contentMap = lib.filterAttrs (_: v: v != null) - (builtins.mapAttrs relevantEntry entries); - in builtins.toJSON contentMap; - - sourceContentHash = builtins.convertHash { - hash = "sha1:${builtins.hashString "sha1" (hashDependencyFiles scrollsDir)}"; - toHashFormat = "base64"; - }; - - # we assume that same lock+config files yield same deno caches here - # however, jsons need to be sorted and some sqlite files deleted - - # jsr gentlemen specifically require their package metadata - # to have multiple timestamps, so they can fuck off - outputHash = knownDenoHashes.${sourceContentHash} or lib.fakeSha256; - - unifiedCache = pkgs.stdenv.mkDerivation { - name = "deno-scrolls-cache"; - src = scrollsDir; - - nativeBuildInputs = [ pkgs.deno pkgs.jq ]; - dontPatchShebangs = true; - - buildPhase = '' - export DENO_DIR=$out - - ${lib.concatStringsSep "\n" (map (pkg: - let config = import (scrollsDir + "/${pkg}/scroll.nix"); - in '' - echo "Caching ${pkg}..." - cd ${pkg} - ${config.cache-command} - cd .. - '' - ) packageDirs)} - - echo 'Go fuck yourself, SQLite!' - find $out -name "*-wal" -delete - find $out -name "*-shm" -delete - - echo 'Go fuck yourself, JSON!' - find $out/npm -name "*.json" -type f 2>/dev/null | while read -r file; do - if ${pkgs.jq}/bin/jq empty "$file" 2>/dev/null; then - ${pkgs.jq}/bin/jq -S . "$file" > "$file.tmp" && mv "$file.tmp" "$file" - fi - done - - echo 'sourceContentHash: ${sourceContentHash}' - echo "" - echo "add me to knownHashes if you see a fake sha256 error ^" - echo "DON'T USE JSR DEPS UNLESS YOU MAKE THEM REPRODUCIBLE SOMEHOW" - echo "" - ''; - - installPhase = "true"; - - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - # this is basically FOD on steroids cause it - # deliberately breaks on relevant source changes - inherit outputHash; - }; - -in -{ - cache = unifiedCache; -} diff --git a/shelf/deno/hjq/deno.lock b/shelf/deno/hjq/deno.lock index e903f12..ea94b66 100644 --- a/shelf/deno/hjq/deno.lock +++ b/shelf/deno/hjq/deno.lock @@ -1,15 +1,26 @@ { - "version": "4", + "version": "5", "specifiers": { + "npm:@types/node@*": "22.13.4", "npm:hjson@^3.2.2": "3.2.2", "npm:jq-web@~0.6.2": "0.6.2" }, "npm": { + "@types/node@22.13.4": { + "integrity": "sha512-ywP2X0DYtX3y08eFVx5fNIw7/uIv8hYUKgXoK8oayJlLnKcRfEYCxWMVE1XagUdVtCJlZT1AU4LXEABW+L1Peg==", + "dependencies": [ + "undici-types" + ] + }, "hjson@3.2.2": { - "integrity": "sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q==" + "integrity": "sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q==", + "bin": true }, "jq-web@0.6.2": { "integrity": "sha512-+7XvjBYwTx4vP5PYkf6Q6orubO/v+UgMU6By1GritrmShr9QpT3UKa4ANzXWQfhdqtBnQYXsm7ZNbdIHT6tYpQ==" + }, + "undici-types@6.20.0": { + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==" } }, "workspace": { diff --git a/shelf/deno/hjq/scroll.nix b/shelf/deno/hjq/scroll.nix index accdc03..dc5d892 100644 --- a/shelf/deno/hjq/scroll.nix +++ b/shelf/deno/hjq/scroll.nix @@ -8,7 +8,7 @@ format = "HJSON"; react = "push"; }; - jq_filter = { + jqFilter = { format = "string"; react = "push"; }; @@ -20,6 +20,13 @@ react = "push"; }; }; - - cache-command = "deno cache --reload main.ts"; + + build = { + hashFiles = [ ./deno.json ./deno.lock ]; + cacheCommand = "deno cache --frozen main.ts"; + knownHashes = { + "sha1-gSeoE0sSj+dFQ7SUoyaQV0X/KJE=" = + "sha256-0ZfdWVfShbhgAoSse9vEpFPRh5XTTVjgxGMmezwon9I="; + }; + }; } diff --git a/shelf/deno/uses-hjq/deno.lock b/shelf/deno/uses-hjq/deno.lock index e9382a1..033d446 100644 --- a/shelf/deno/uses-hjq/deno.lock +++ b/shelf/deno/uses-hjq/deno.lock @@ -1,15 +1,26 @@ { - "version": "4", + "version": "5", "specifiers": { + "npm:@types/node@*": "22.13.4", "npm:hjson@^3.2.2": "3.2.2", "npm:jq-web@~0.6.2": "0.6.2" }, "npm": { + "@types/node@22.13.4": { + "integrity": "sha512-ywP2X0DYtX3y08eFVx5fNIw7/uIv8hYUKgXoK8oayJlLnKcRfEYCxWMVE1XagUdVtCJlZT1AU4LXEABW+L1Peg==", + "dependencies": [ + "undici-types" + ] + }, "hjson@3.2.2": { - "integrity": "sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q==" + "integrity": "sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q==", + "bin": true }, "jq-web@0.6.2": { "integrity": "sha512-+7XvjBYwTx4vP5PYkf6Q6orubO/v+UgMU6By1GritrmShr9QpT3UKa4ANzXWQfhdqtBnQYXsm7ZNbdIHT6tYpQ==" + }, + "undici-types@6.20.0": { + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==" } } } diff --git a/shelf/deno/uses-hjq/scroll.nix b/shelf/deno/uses-hjq/scroll.nix index 5faee5e..448d48d 100644 --- a/shelf/deno/uses-hjq/scroll.nix +++ b/shelf/deno/uses-hjq/scroll.nix @@ -1,3 +1,32 @@ { - cache-command = "deno cache -I --reload main.ts"; + name = "uses-hjq"; + + description = "A test scroll that mimicks hjq's behavior"; + + inputs = { + stdin = { + format = "HJSON"; + react = "push"; + }; + jqFilter = { + format = "string"; + react = "push"; + }; + }; + + outputs = { + stdout = { + format = "HJSON"; + react = "push"; + }; + }; + + build = { + hashFiles = [ ./deno.json ./deno.lock ]; + cacheCommand = "deno cache --frozen main.ts"; + knownHashes = { + "sha1-uO43Rt1F6+Ud1wk2p9LQhfP180M=" = + "sha256-x/Fvn/l3J7C0D7AftlbRk7iuc6o9qPakdnHb72TSxec="; + } ; + }; }