From 479241a80e636205cb13068e17f834b82a04c1a5 Mon Sep 17 00:00:00 2001 From: Max Horn Date: Wed, 12 Aug 2026 01:31:27 +0200 Subject: [PATCH 1/2] kernel: add HexSHA256File Checksumming a file so far meant HexSHA256(InputTextFile(name)), which reads the whole file into memory, and reads it as text: a file whose name ends in '.gz' is silently decompressed, and on systems distinguishing text and binary mode the line endings are translated. Neither is what one wants when checking a downloaded file against a published checksum. HexSHA256File reads the file in binary and in chunks, so the digest describes the bytes on disk and the file size is not bounded by memory. Passing 'true' as second argument opts into the transparent decompression instead, since SyFopen offers it anyway. HexSHA256 on a stream now also hashes in chunks rather than calling ReadAll; the comment claiming the streams API cannot do this predates ReadAll gaining its length argument. Assistance from Claude Opus 5 via Claude Code: implementation, documentation and tests, reviewed by me. Co-authored-by: Claude Opus 5 --- doc/ref/string.xml | 1 + lib/files.gd | 37 +++++++++++++++++++++ lib/files.gi | 66 +++++++++++++++++++++++++++++--------- src/sha256.c | 40 +++++++++++++++++++++++ tst/testinstall/sha256.tst | 43 +++++++++++++++++++++++++ 5 files changed, 171 insertions(+), 16 deletions(-) diff --git a/doc/ref/string.xml b/doc/ref/string.xml index 2bd0e41943..69e815649d 100644 --- a/doc/ref/string.xml +++ b/doc/ref/string.xml @@ -691,6 +691,7 @@ gap> HexStringInt(last); <#Include Label="EvalString"> <#Include Label="CrcString"> <#Include Label="HexSHA256"> +<#Include Label="HexSHA256File"> <#Include Label="Pluralize"> diff --git a/lib/files.gd b/lib/files.gd index 5e62bcc54a..586c234c0b 100644 --- a/lib/files.gd +++ b/lib/files.gd @@ -850,5 +850,42 @@ DeclareGlobalFunction( "Edit" ); ## DeclareGlobalFunction("HexSHA256"); + +## <#GAPDoc Label="HexSHA256File"> +## +## +## +## +## hash function +## checksum +## Return the SHA-256 cryptographic checksum of the contents of the file +## filename, as a string with 64 lowercase hexadecimal digits, +## or fail if the file cannot be read. +##

+## The file is read as binary data, so the result always describes the bytes +## on disk, also on systems which distinguish text and binary mode and would +## otherwise translate line endings. It is read in chunks, so the size of +## the file is not limited by the available memory. +##

+## If the optional argument decompress is true, then a file +## whose name ends in .gz is decompressed while reading, so that the +## checksum describes the decompressed data, as read by +## . +## The default is false, which is what one wants in order to compare +## against a checksum published alongside a file. +## name := Filename(DirectoryTemporary(), "test.txt");; +## gap> FileString(name, "abcd");; +## gap> HexSHA256File(name); +## "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589" +## gap> HexSHA256File("/no/such/file"); +## fail +## ]]> +## +## +## <#/GAPDoc> +## +DeclareGlobalFunction("HexSHA256File"); + BIND_GLOBAL("GAP_SHA256_State_Type", NewType(NewFamily("GAP_SHA256_State_Family"), IsObject) ); diff --git a/lib/files.gi b/lib/files.gi index 64e0afa5ac..e356a0d77f 100644 --- a/lib/files.gi +++ b/lib/files.gi @@ -393,30 +393,64 @@ InstallGlobalFunction(RemoveDirectoryRecursively, return Dowork(dirname); end ); +BindGlobal( "GAP_SHA256_HexOfWords", +function(words) + local res; + + res := Sum([0..7], i -> words[8-i]*2^(32*i)); + res := LowercaseString(HexStringInt(res)); + # HexStringInt drops leading zero digits, but a SHA256 digest is always + # 256 bits = 64 hex digits, so left-pad with '0' if the top byte(s) were 0. + if Length(res) < 64 then + res := Concatenation(ListWithIdenticalEntries(64 - Length(res), '0'), res); + fi; + return res; +end); + InstallGlobalFunction( HexSHA256, function(str) - local s, res; + local s, chunk; + s := GAP_SHA256_INIT(); if IsString(str) then - str := CopyToStringRep(str); + GAP_SHA256_UPDATE(s, CopyToStringRep(str)); elif IsInputStream(str) then - str := ReadAll(str); - # TODO: instead o reading the complete stream at once (which might be - # huge), it would be better to read it in chunks, say 16kb at a time. - # Alas, our streams API currently offers no way to do that. + # read in chunks: the stream may deliver more than fits in memory + repeat + chunk := ReadAll(str, 65536); + if IsString(chunk) and Length(chunk) > 0 then + GAP_SHA256_UPDATE(s, CopyToStringRep(chunk)); + fi; + until not IsString(chunk) or Length(chunk) = 0; else ErrorNoReturn(" has to be a string or an input stream"); fi; - s := GAP_SHA256_INIT(); - GAP_SHA256_UPDATE(s, str); - res := GAP_SHA256_FINAL(s); - res := Sum([0..7], i -> res[8-i]*2^(32*i));; - res := LowercaseString(HexStringInt(res)); - # HexStringInt drops leading zero digits, but a SHA256 digest is always - # 256 bits = 64 hex digits, so left-pad with '0' if the top byte(s) were 0. - if Length(res) < 64 then - res := Concatenation(ListWithIdenticalEntries(64 - Length(res), '0'), res); + return GAP_SHA256_HexOfWords(GAP_SHA256_FINAL(s)); +end); + +InstallGlobalFunction( HexSHA256File, +function(args...) + local filename, decompress, res; + + if Length(args) = 0 or 2 < Length(args) then + ErrorNoReturn("usage: HexSHA256File( [, ] )"); fi; - return res; + filename := args[1]; + if not IsString(filename) then + ErrorNoReturn(" must be a string"); + fi; + decompress := false; + if Length(args) = 2 then + if not args[2] in [ true, false ] then + ErrorNoReturn(" must be 'true' or 'false'"); + fi; + decompress := args[2]; + fi; + + res := GAP_SHA256_FILE(UserHomeExpand(filename), decompress); + if res = fail then + return fail; + fi; + return GAP_SHA256_HexOfWords(res); end); diff --git a/src/sha256.c b/src/sha256.c index 65786f1bc3..48aa5abbee 100644 --- a/src/sha256.c +++ b/src/sha256.c @@ -17,6 +17,7 @@ #include "objects.h" #include "plist.h" #include "stringobj.h" +#include "sysfiles.h" #include "config.h" @@ -295,6 +296,44 @@ Obj FuncGAP_SHA256_FINAL(Obj self, Obj state) return result; } +Obj FuncGAP_SHA256_FILE(Obj self, Obj filename, Obj decompress) +{ + Obj result; + sha256_state_t st; + Int fid, len; + int i; + UChar buf[16384]; + + RequireStringRep(SELF_NAME, filename); + RequireTrueOrFalse(SELF_NAME, decompress); + + // Mode "rb" rather than "r" so that no line ending translation happens; + // the digest has to describe the bytes on disk. With set, + // a file whose name ends in '.gz' is hashed as its decompressed content + // instead, matching what 'InputTextFile' would read. + fid = SyFopen(CONST_CSTR_STRING(filename), "rb", decompress == True); + if (fid == -1) + return Fail; + + sha256_init(&st); + while ((len = SyRead(fid, buf, sizeof(buf))) > 0) { + sha256_update(&st, buf, len); + } + SyFclose(fid); + if (len < 0) + return Fail; + + sha256_final(&st); + + result = NEW_PLIST(T_PLIST, 8); + SET_LEN_PLIST(result, 8); + for (i = 0; i < 8; i++) { + SET_ELM_PLIST(result, i + 1, ObjInt_UInt(st.r[i])); + CHANGED_BAG(result); + } + return result; +} + Obj FuncGAP_SHA256_HMAC(Obj self, Obj key, Obj text) { UInt i, klen; @@ -355,6 +394,7 @@ static StructGVarFunc GVarFuncs[] = { GVAR_FUNC_0ARGS(GAP_SHA256_INIT), GVAR_FUNC_2ARGS(GAP_SHA256_UPDATE, state, bytes), GVAR_FUNC_1ARGS(GAP_SHA256_FINAL, state), + GVAR_FUNC_2ARGS(GAP_SHA256_FILE, filename, decompress), GVAR_FUNC_2ARGS(GAP_SHA256_HMAC, key, text), { 0 } // Finish with an empty entry diff --git a/tst/testinstall/sha256.tst b/tst/testinstall/sha256.tst index 7f6988cb3a..7af0a0528d 100644 --- a/tst/testinstall/sha256.tst +++ b/tst/testinstall/sha256.tst @@ -1,4 +1,5 @@ # +#@local dir, gzname, name, out, state, str gap> START_TEST("sha256.tst"); # @@ -57,5 +58,47 @@ gap> HexSHA256(InputTextString("abcd\r\n")); gap> HexSHA256(InputTextString("")); "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +# HexSHA256File +gap> dir := DirectoryTemporary();; +gap> name := Filename(dir, "test.txt");; +gap> FileString(name, "abcd");; +gap> HexSHA256File(name); +"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589" +gap> FileString(Filename(dir, "empty.txt"), "");; +gap> HexSHA256File(Filename(dir, "empty.txt")); +"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +# larger than the read buffer, so that more than one chunk gets hashed +gap> str := Concatenation(List([1 .. 5000], i -> "0123456789"));; +gap> FileString(Filename(dir, "big.txt"), str);; +gap> HexSHA256File(Filename(dir, "big.txt")) = HexSHA256(str); +true + +# a file that is not there +gap> HexSHA256File(Filename(dir, "no-such-file")); +fail + +# '.gz' files are hashed as they are on disk unless asked otherwise +gap> gzname := Filename(dir, "compressed.txt.gz");; +gap> out := OutputGzipFile(gzname, false);; +gap> WriteAll(out, "abcd");; +gap> CloseStream(out); +gap> HexSHA256File(gzname) = HexSHA256("abcd"); +false +gap> HexSHA256File(gzname, true) = HexSHA256("abcd"); +true +gap> HexSHA256File(gzname, false) = HexSHA256File(gzname); +true + +# argument checking +gap> HexSHA256File(); +Error, usage: HexSHA256File( [, ] ) +gap> HexSHA256File(name, true, true); +Error, usage: HexSHA256File( [, ] ) +gap> HexSHA256File(42); +Error, must be a string +gap> HexSHA256File(name, "yes"); +Error, must be 'true' or 'false' + # gap> STOP_TEST("sha256.tst"); From 9c88d91a18709b3c85ba953d3b56cd516af26822 Mon Sep 17 00:00:00 2001 From: Max Horn Date: Thu, 13 Aug 2026 18:52:13 +0200 Subject: [PATCH 2/2] Make the decompress argument of HexSHA256File required Everything else in GAP decompresses a '.gz' file as it reads it, so a default of 'false' here is the one function that quietly does the opposite. Requiring the argument makes the choice visible at every call site, and it removes the optional-argument handling entirely. Co-Authored-By: Claude Opus 5 --- lib/files.gd | 20 +++++++++++--------- lib/files.gi | 17 ++++------------- tst/testinstall/sha256.tst | 22 +++++++++------------- 3 files changed, 24 insertions(+), 35 deletions(-) diff --git a/lib/files.gd b/lib/files.gd index 586c234c0b..4f5bf23ee4 100644 --- a/lib/files.gd +++ b/lib/files.gd @@ -853,7 +853,7 @@ DeclareGlobalFunction("HexSHA256"); ## <#GAPDoc Label="HexSHA256File"> ## -## +## ## ## ## hash function @@ -867,18 +867,20 @@ DeclareGlobalFunction("HexSHA256"); ## otherwise translate line endings. It is read in chunks, so the size of ## the file is not limited by the available memory. ##

-## If the optional argument decompress is true, then a file -## whose name ends in .gz is decompressed while reading, so that the -## checksum describes the decompressed data, as read by -## . -## The default is false, which is what one wants in order to compare -## against a checksum published alongside a file. +## decompress must be true or false, and says which of +## two different checksums is wanted: with true, a file whose name +## ends in .gz is decompressed while reading, so that the checksum +## describes the data as would +## read it; with false, it describes the bytes on disk, which is what +## one wants in order to compare against a checksum published alongside a +## file. There is no default, because everything else in &GAP; decompresses +## and silently doing the opposite would be a trap. ## name := Filename(DirectoryTemporary(), "test.txt");; ## gap> FileString(name, "abcd");; -## gap> HexSHA256File(name); +## gap> HexSHA256File(name, false); ## "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589" -## gap> HexSHA256File("/no/such/file"); +## gap> HexSHA256File("/no/such/file", false); ## fail ## ]]> ## diff --git a/lib/files.gi b/lib/files.gi index e356a0d77f..90761d325e 100644 --- a/lib/files.gi +++ b/lib/files.gi @@ -430,22 +430,13 @@ function(str) end); InstallGlobalFunction( HexSHA256File, -function(args...) - local filename, decompress, res; +function(filename, decompress) + local res; - if Length(args) = 0 or 2 < Length(args) then - ErrorNoReturn("usage: HexSHA256File( [, ] )"); - fi; - filename := args[1]; if not IsString(filename) then ErrorNoReturn(" must be a string"); - fi; - decompress := false; - if Length(args) = 2 then - if not args[2] in [ true, false ] then - ErrorNoReturn(" must be 'true' or 'false'"); - fi; - decompress := args[2]; + elif not decompress in [ true, false ] then + ErrorNoReturn(" must be 'true' or 'false'"); fi; res := GAP_SHA256_FILE(UserHomeExpand(filename), decompress); diff --git a/tst/testinstall/sha256.tst b/tst/testinstall/sha256.tst index 7af0a0528d..7350ea87ba 100644 --- a/tst/testinstall/sha256.tst +++ b/tst/testinstall/sha256.tst @@ -62,40 +62,36 @@ gap> HexSHA256(InputTextString("")); gap> dir := DirectoryTemporary();; gap> name := Filename(dir, "test.txt");; gap> FileString(name, "abcd");; -gap> HexSHA256File(name); +gap> HexSHA256File(name, false); "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589" gap> FileString(Filename(dir, "empty.txt"), "");; -gap> HexSHA256File(Filename(dir, "empty.txt")); +gap> HexSHA256File(Filename(dir, "empty.txt"), false); "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" # larger than the read buffer, so that more than one chunk gets hashed gap> str := Concatenation(List([1 .. 5000], i -> "0123456789"));; gap> FileString(Filename(dir, "big.txt"), str);; -gap> HexSHA256File(Filename(dir, "big.txt")) = HexSHA256(str); +gap> HexSHA256File(Filename(dir, "big.txt"), false) = HexSHA256(str); true # a file that is not there -gap> HexSHA256File(Filename(dir, "no-such-file")); +gap> HexSHA256File(Filename(dir, "no-such-file"), false); fail -# '.gz' files are hashed as they are on disk unless asked otherwise +# the two answers a '.gz' file has gap> gzname := Filename(dir, "compressed.txt.gz");; gap> out := OutputGzipFile(gzname, false);; gap> WriteAll(out, "abcd");; gap> CloseStream(out); -gap> HexSHA256File(gzname) = HexSHA256("abcd"); +gap> HexSHA256File(gzname, false) = HexSHA256("abcd"); false gap> HexSHA256File(gzname, true) = HexSHA256("abcd"); true -gap> HexSHA256File(gzname, false) = HexSHA256File(gzname); -true # argument checking -gap> HexSHA256File(); -Error, usage: HexSHA256File( [, ] ) -gap> HexSHA256File(name, true, true); -Error, usage: HexSHA256File( [, ] ) -gap> HexSHA256File(42); +gap> HexSHA256File(name); +Error, Function: number of arguments must be 2 (not 1) +gap> HexSHA256File(42, false); Error, must be a string gap> HexSHA256File(name, "yes"); Error, must be 'true' or 'false'