voleum/gr-base
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

deno per package cache derivations + hashes

Browse source
This commit is contained in:
EatThePooh 2025-08-02 14:14:41 +07:00
parent 9c18a0c924
commit f459ec8b78
9 changed files with 203 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

14
flake.nix
View file

@ -10,16 +10,24 @@
flake-utils.lib.eachDefaultSystem (system: flake-utils.lib.eachDefaultSystem (system:
let let
pkgs = nixpkgs.legacyPackages.${system}; pkgs = nixpkgs.legacyPackages.${system};
lib = {
collectDenoCaches = import ./nix/deno/collect_caches.nix;
readDenoScroll = import ./nix/deno/read_scroll.nix;
};
shelf = import ./shelf { shelf = import ./shelf {
inherit pkgs system; inherit pkgs system lib;
}; };
in in
{ {
packages.shelf = shelf; inherit lib;
devShells.default = pkgs.mkShell { devShells.default = pkgs.mkShell {
buildInputs = [ ]; buildInputs = [ pkgs.deno ];
shellHook = '' shellHook = ''
${shelf.setupScript} ${shelf.setupScript}
echo "Availabe scrolls:"
echo ${pkgs.lib.escapeShellArg (builtins.toJSON shelf.scrolls)} | jq .
''; '';
}; };
}); });

42
nix/deno/collect_caches.nix Normal file
View file

@ -0,0 +1,42 @@
{ pkgs, denoCacheDrvs }:
let
mergeOneCache = cache: ''
if [ -d "${cache}" ]; then
echo " Merging cache from ${cache}"
# Use cp with fallback, handle overlapping files gracefully
cp -rL "${cache}"/* "$out/" 2>/dev/null || true
else
echo " Warning: ${cache} is not a directory, skipping"
fi
'';
mergeCacheCmd = pkgs.runCommand "deno-shelf-shared-cache"
{
buildInputs = denoCacheDrvs;
} ''
mkdir -p $out
echo "Merging ${builtins.toString (builtins.length denoCacheDrvs)} Deno caches..."
${pkgs.lib.concatMapStringsSep "\n" mergeOneCache denoCacheDrvs}
echo "Deno caches merged. Final size: $(du -sh $out | cut -f1)"
'';
in
pkgs.writeShellScript "deno-shelf-setup" ''
export DENO_DIR=$PWD/.deno_cache
if [ ! -d "$DENO_DIR" ] || [ "${mergeCacheCmd}" -nt "$DENO_DIR/.cache_timestamp" ]; then
echo "Setting up mutable Deno cache..."
rm -rf "$DENO_DIR"
cp -r "${mergeCacheCmd}" "$DENO_DIR"
chmod -R u+w "$DENO_DIR"
touch "$DENO_DIR/.cache_timestamp"
echo "Deno cache initialized"
else
echo "Deno cache already up to date"
fi
''

64
nix/deno/read_scroll.nix Normal file
View file

@ -0,0 +1,64 @@
{ pkgs, system, scrollsDir, subDir }:
let
target =
let
arch = builtins.head (builtins.split "-" system);
os = builtins.elemAt (builtins.split "-" system) 1;
vendor = if os == "darwin" then "apple" else "unknown";
sys = if os == "darwin" then "darwin" else "linux-gnu";
in "${arch}-${vendor}-${sys}";
scroll = import "${scrollsDir}/${subDir}/scroll.nix";
hashFilesCat = builtins.concatStringsSep ""
(builtins.map builtins.readFile scroll.build.hashFiles);
sourceContentHash_ = builtins.convertHash {
hash = "sha1:${builtins.hashString "sha1" hashFilesCat}";
toHashFormat = "base64";
};
sourceContentHash = "sha1-${sourceContentHash_}";
outputHash =
scroll.build.knownHashes.${sourceContentHash}
or pkgs.lib.fakeSha256;
in
{
env = pkgs.stdenv.mkDerivation {
name = "${scroll.name}-scroll-env";
src = scrollsDir;
nativeBuildInputs = [ pkgs.deno pkgs.jq ];
dontPatchShebangs = true;
buildPhase = ''
export DENO_DIR=$out
cd ${subDir}
${scroll.build.cacheCommand}
cd ..
echo 'Go fuck yourself, SQLite!'
find $out -name "*-wal" -delete
find $out -name "*-shm" -delete
echo 'Go fuck yourself, JSON!'
find $out/npm -name "*.json" -type f 2>/dev/null | while read -r file; do
if ${pkgs.jq}/bin/jq empty "$file" 2>/dev/null; then
${pkgs.jq}/bin/jq -S . "$file" > "$file.tmp" && mv "$file.tmp" "$file"
fi
done
echo 'sourceContentHash: ${sourceContentHash}'
echo ""
echo "add me to knownHashes if you see a fake sha256 error ^"
echo "DON'T USE JSR DEPS UNLESS YOU MAKE THEM REPRODUCIBLE SOMEHOW"
echo ""
'';
installPhase = "true";
outputHashMode = "recursive";
outputHashAlgo = "sha256";
inherit outputHash;
};
meta = builtins.removeAttrs scroll [ "build" ];
}

39
shelf/default.nix
View file

@ -1,24 +1,25 @@
{ pkgs, system }: { pkgs, system, lib }:
let let
denoShelf = import ./deno { denoScrollsDir = ./deno;
inherit pkgs system;
# collective hash of all (relevant) deno sources
# mapped to resulting shared deno cache hash
knownDenoHashes = {
"2JQI9Ie/fkl5Ltr22BGJynF9tFc=" =
"sha256-7csbrTbGd6J4a0Gkj9ndKhNxv90KeOHjr7DDNal3L4Q=";
};
};
shelfSetup = pkgs.writeShellScript "setup-shelf" ''
export DENO_DIR=$PWD/.deno_cache
rm -rf $DENO_DIR
cp -r ${denoShelf.cache} $DENO_DIR
chmod -R u+w "$DENO_DIR"
# TODO: add other interpreters mkScroll = subDir: lib.readDenoScroll {
''; inherit pkgs system subDir;
scrollsDir = denoScrollsDir;
};
scrolls = builtins.map mkScroll [
"hjq"
"uses-hjq"
];
shelfSetup =
lib.collectDenoCaches {
inherit pkgs;
denoCacheDrvs = builtins.map (s: s.env) scrolls;
};
in in
{ {
setupScript = shelfSetup; setupScript = shelfSetup;
scrolls = builtins.map (s: s.meta) scrolls;
} }

103
shelf/deno/default.nix
View file

@ -1,103 +0,0 @@
{ pkgs, system, knownDenoHashes }:
let
target =
let
arch = builtins.head (builtins.split "-" system);
os = builtins.elemAt (builtins.split "-" system) 1;
vendor = if os == "darwin" then "apple" else "unknown";
sys = if os == "darwin" then "darwin" else "linux-gnu";
in "${arch}-${vendor}-${sys}";
# currently unused, needed for deno compile
denort = pkgs.fetchzip {
url = "https://dl.deno.land/release/v${pkgs.deno.version}/denort-${target}.zip";
hash = "sha256-ukIk8K2CE+N+3eFs++RPiGZlhhRRVk1gjhdt77/s+4o=";
};
scrollsDir = ./.;
lib = pkgs.lib;
packageDirs = [ "./hjq" "./uses-hjq" ];
# hash only dependency-relevant files (lock files and configs)
hashDependencyFiles = dir:
let
entries = builtins.readDir dir;
# Only process directories and dependency-relevant files
relevantEntry = name: type:
if type == "directory"
then hashDependencyFiles (dir + "/${name}")
else if (lib.hasSuffix ".lock" name ||
lib.hasSuffix ".json" name ||
lib.hasSuffix ".jsonc" name)
then builtins.readFile (dir + "/${name}")
else null;
contentMap = lib.filterAttrs (_: v: v != null)
(builtins.mapAttrs relevantEntry entries);
in builtins.toJSON contentMap;
sourceContentHash = builtins.convertHash {
hash = "sha1:${builtins.hashString "sha1" (hashDependencyFiles scrollsDir)}";
toHashFormat = "base64";
};
# we assume that same lock+config files yield same deno caches here
# however, jsons need to be sorted and some sqlite files deleted
# jsr gentlemen specifically require their package metadata
# to have multiple timestamps, so they can fuck off
outputHash = knownDenoHashes.${sourceContentHash} or lib.fakeSha256;
unifiedCache = pkgs.stdenv.mkDerivation {
name = "deno-scrolls-cache";
src = scrollsDir;
nativeBuildInputs = [ pkgs.deno pkgs.jq ];
dontPatchShebangs = true;
buildPhase = ''
export DENO_DIR=$out
${lib.concatStringsSep "\n" (map (pkg:
let config = import (scrollsDir + "/${pkg}/scroll.nix");
in ''
echo "Caching ${pkg}..."
cd ${pkg}
${config.cache-command}
cd ..
''
) packageDirs)}
echo 'Go fuck yourself, SQLite!'
find $out -name "*-wal" -delete
find $out -name "*-shm" -delete
echo 'Go fuck yourself, JSON!'
find $out/npm -name "*.json" -type f 2>/dev/null | while read -r file; do
if ${pkgs.jq}/bin/jq empty "$file" 2>/dev/null; then
${pkgs.jq}/bin/jq -S . "$file" > "$file.tmp" && mv "$file.tmp" "$file"
fi
done
echo 'sourceContentHash: ${sourceContentHash}'
echo ""
echo "add me to knownHashes if you see a fake sha256 error ^"
echo "DON'T USE JSR DEPS UNLESS YOU MAKE THEM REPRODUCIBLE SOMEHOW"
echo ""
'';
installPhase = "true";
outputHashMode = "recursive";
outputHashAlgo = "sha256";
# this is basically FOD on steroids cause it
# deliberately breaks on relevant source changes
inherit outputHash;
};
in
{
cache = unifiedCache;
}

15
shelf/deno/hjq/deno.lock generated
View file

@ -1,15 +1,26 @@
{ {
"version": "4", "version": "5",
"specifiers": { "specifiers": {
"npm:@types/node@*": "22.13.4",
"npm:hjson@^3.2.2": "3.2.2", "npm:hjson@^3.2.2": "3.2.2",
"npm:jq-web@~0.6.2": "0.6.2" "npm:jq-web@~0.6.2": "0.6.2"
}, },
"npm": { "npm": {
"@types/node@22.13.4": {
"integrity": "sha512-ywP2X0DYtX3y08eFVx5fNIw7/uIv8hYUKgXoK8oayJlLnKcRfEYCxWMVE1XagUdVtCJlZT1AU4LXEABW+L1Peg==",
"dependencies": [
"undici-types"
]
},
"hjson@3.2.2": { "hjson@3.2.2": {
"integrity": "sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q==" "integrity": "sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q==",
"bin": true
}, },
"jq-web@0.6.2": { "jq-web@0.6.2": {
"integrity": "sha512-+7XvjBYwTx4vP5PYkf6Q6orubO/v+UgMU6By1GritrmShr9QpT3UKa4ANzXWQfhdqtBnQYXsm7ZNbdIHT6tYpQ==" "integrity": "sha512-+7XvjBYwTx4vP5PYkf6Q6orubO/v+UgMU6By1GritrmShr9QpT3UKa4ANzXWQfhdqtBnQYXsm7ZNbdIHT6tYpQ=="
},
"undici-types@6.20.0": {
"integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="
} }
}, },
"workspace": { "workspace": {

13
shelf/deno/hjq/scroll.nix
View file

@ -8,7 +8,7 @@
format = "HJSON"; format = "HJSON";
react = "push"; react = "push";
}; };
jq_filter = { jqFilter = {
format = "string"; format = "string";
react = "push"; react = "push";
}; };
@ -20,6 +20,13 @@
react = "push"; react = "push";
}; };
}; };
cache-command = "deno cache --reload main.ts"; build = {
hashFiles = [ ./deno.json ./deno.lock ];
cacheCommand = "deno cache --frozen main.ts";
knownHashes = {
"sha1-gSeoE0sSj+dFQ7SUoyaQV0X/KJE=" =
"sha256-0ZfdWVfShbhgAoSse9vEpFPRh5XTTVjgxGMmezwon9I=";
};
};
} }

15
shelf/deno/uses-hjq/deno.lock generated
View file

@ -1,15 +1,26 @@
{ {
"version": "4", "version": "5",
"specifiers": { "specifiers": {
"npm:@types/node@*": "22.13.4",
"npm:hjson@^3.2.2": "3.2.2", "npm:hjson@^3.2.2": "3.2.2",
"npm:jq-web@~0.6.2": "0.6.2" "npm:jq-web@~0.6.2": "0.6.2"
}, },
"npm": { "npm": {
"@types/node@22.13.4": {
"integrity": "sha512-ywP2X0DYtX3y08eFVx5fNIw7/uIv8hYUKgXoK8oayJlLnKcRfEYCxWMVE1XagUdVtCJlZT1AU4LXEABW+L1Peg==",
"dependencies": [
"undici-types"
]
},
"hjson@3.2.2": { "hjson@3.2.2": {
"integrity": "sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q==" "integrity": "sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q==",
"bin": true
}, },
"jq-web@0.6.2": { "jq-web@0.6.2": {
"integrity": "sha512-+7XvjBYwTx4vP5PYkf6Q6orubO/v+UgMU6By1GritrmShr9QpT3UKa4ANzXWQfhdqtBnQYXsm7ZNbdIHT6tYpQ==" "integrity": "sha512-+7XvjBYwTx4vP5PYkf6Q6orubO/v+UgMU6By1GritrmShr9QpT3UKa4ANzXWQfhdqtBnQYXsm7ZNbdIHT6tYpQ=="
},
"undici-types@6.20.0": {
"integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="
} }
} }
} }

31
shelf/deno/uses-hjq/scroll.nix
View file

@ -1,3 +1,32 @@
{ {
cache-command = "deno cache -I --reload main.ts"; name = "uses-hjq";
description = "A test scroll that mimicks hjq's behavior";
inputs = {
stdin = {
format = "HJSON";
react = "push";
};
jqFilter = {
format = "string";
react = "push";
};
};
outputs = {
stdout = {
format = "HJSON";
react = "push";
};
};
build = {
hashFiles = [ ./deno.json ./deno.lock ];
cacheCommand = "deno cache --frozen main.ts";
knownHashes = {
"sha1-uO43Rt1F6+Ud1wk2p9LQhfP180M=" =
"sha256-x/Fvn/l3J7C0D7AftlbRk7iuc6o9qPakdnHb72TSxec=";
} ;
};
} }