diff --git a/doc/ref/string.xml b/doc/ref/string.xml index 2bd0e41943..69e815649d 100644 --- a/doc/ref/string.xml +++ b/doc/ref/string.xml @@ -691,6 +691,7 @@ gap> HexStringInt(last); <#Include Label="EvalString"> <#Include Label="CrcString"> <#Include Label="HexSHA256"> +<#Include Label="HexSHA256File"> <#Include Label="Pluralize"> diff --git a/lib/files.gd b/lib/files.gd index 5e62bcc54a..4f5bf23ee4 100644 --- a/lib/files.gd +++ b/lib/files.gd @@ -850,5 +850,44 @@ DeclareGlobalFunction( "Edit" ); ## DeclareGlobalFunction("HexSHA256"); + +## <#GAPDoc Label="HexSHA256File"> +## +## +## +## +## hash function +## checksum +## Return the SHA-256 cryptographic checksum of the contents of the file +## filename, as a string with 64 lowercase hexadecimal digits, +## or fail if the file cannot be read. +##

+## The file is read as binary data, so the result always describes the bytes +## on disk, also on systems which distinguish text and binary mode and would +## otherwise translate line endings. It is read in chunks, so the size of +## the file is not limited by the available memory. +##

+## decompress must be true or false, and says which of +## two different checksums is wanted: with true, a file whose name +## ends in .gz is decompressed while reading, so that the checksum +## describes the data as would +## read it; with false, it describes the bytes on disk, which is what +## one wants in order to compare against a checksum published alongside a +## file. There is no default, because everything else in &GAP; decompresses +## and silently doing the opposite would be a trap. +## name := Filename(DirectoryTemporary(), "test.txt");; +## gap> FileString(name, "abcd");; +## gap> HexSHA256File(name, false); +## "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589" +## gap> HexSHA256File("/no/such/file", false); +## fail +## ]]> +## +## +## <#/GAPDoc> +## +DeclareGlobalFunction("HexSHA256File"); + BIND_GLOBAL("GAP_SHA256_State_Type", NewType(NewFamily("GAP_SHA256_State_Family"), IsObject) ); diff --git a/lib/files.gi b/lib/files.gi index 64e0afa5ac..90761d325e 100644 --- a/lib/files.gi +++ b/lib/files.gi @@ -393,30 +393,55 @@ InstallGlobalFunction(RemoveDirectoryRecursively, return Dowork(dirname); end ); +BindGlobal( "GAP_SHA256_HexOfWords", +function(words) + local res; + + res := Sum([0..7], i -> words[8-i]*2^(32*i)); + res := LowercaseString(HexStringInt(res)); + # HexStringInt drops leading zero digits, but a SHA256 digest is always + # 256 bits = 64 hex digits, so left-pad with '0' if the top byte(s) were 0. + if Length(res) < 64 then + res := Concatenation(ListWithIdenticalEntries(64 - Length(res), '0'), res); + fi; + return res; +end); + InstallGlobalFunction( HexSHA256, function(str) - local s, res; + local s, chunk; + s := GAP_SHA256_INIT(); if IsString(str) then - str := CopyToStringRep(str); + GAP_SHA256_UPDATE(s, CopyToStringRep(str)); elif IsInputStream(str) then - str := ReadAll(str); - # TODO: instead o reading the complete stream at once (which might be - # huge), it would be better to read it in chunks, say 16kb at a time. - # Alas, our streams API currently offers no way to do that. + # read in chunks: the stream may deliver more than fits in memory + repeat + chunk := ReadAll(str, 65536); + if IsString(chunk) and Length(chunk) > 0 then + GAP_SHA256_UPDATE(s, CopyToStringRep(chunk)); + fi; + until not IsString(chunk) or Length(chunk) = 0; else ErrorNoReturn(" has to be a string or an input stream"); fi; - s := GAP_SHA256_INIT(); - GAP_SHA256_UPDATE(s, str); - res := GAP_SHA256_FINAL(s); - res := Sum([0..7], i -> res[8-i]*2^(32*i));; - res := LowercaseString(HexStringInt(res)); - # HexStringInt drops leading zero digits, but a SHA256 digest is always - # 256 bits = 64 hex digits, so left-pad with '0' if the top byte(s) were 0. - if Length(res) < 64 then - res := Concatenation(ListWithIdenticalEntries(64 - Length(res), '0'), res); + return GAP_SHA256_HexOfWords(GAP_SHA256_FINAL(s)); +end); + +InstallGlobalFunction( HexSHA256File, +function(filename, decompress) + local res; + + if not IsString(filename) then + ErrorNoReturn(" must be a string"); + elif not decompress in [ true, false ] then + ErrorNoReturn(" must be 'true' or 'false'"); fi; - return res; + + res := GAP_SHA256_FILE(UserHomeExpand(filename), decompress); + if res = fail then + return fail; + fi; + return GAP_SHA256_HexOfWords(res); end); diff --git a/src/sha256.c b/src/sha256.c index 65786f1bc3..48aa5abbee 100644 --- a/src/sha256.c +++ b/src/sha256.c @@ -17,6 +17,7 @@ #include "objects.h" #include "plist.h" #include "stringobj.h" +#include "sysfiles.h" #include "config.h" @@ -295,6 +296,44 @@ Obj FuncGAP_SHA256_FINAL(Obj self, Obj state) return result; } +Obj FuncGAP_SHA256_FILE(Obj self, Obj filename, Obj decompress) +{ + Obj result; + sha256_state_t st; + Int fid, len; + int i; + UChar buf[16384]; + + RequireStringRep(SELF_NAME, filename); + RequireTrueOrFalse(SELF_NAME, decompress); + + // Mode "rb" rather than "r" so that no line ending translation happens; + // the digest has to describe the bytes on disk. With set, + // a file whose name ends in '.gz' is hashed as its decompressed content + // instead, matching what 'InputTextFile' would read. + fid = SyFopen(CONST_CSTR_STRING(filename), "rb", decompress == True); + if (fid == -1) + return Fail; + + sha256_init(&st); + while ((len = SyRead(fid, buf, sizeof(buf))) > 0) { + sha256_update(&st, buf, len); + } + SyFclose(fid); + if (len < 0) + return Fail; + + sha256_final(&st); + + result = NEW_PLIST(T_PLIST, 8); + SET_LEN_PLIST(result, 8); + for (i = 0; i < 8; i++) { + SET_ELM_PLIST(result, i + 1, ObjInt_UInt(st.r[i])); + CHANGED_BAG(result); + } + return result; +} + Obj FuncGAP_SHA256_HMAC(Obj self, Obj key, Obj text) { UInt i, klen; @@ -355,6 +394,7 @@ static StructGVarFunc GVarFuncs[] = { GVAR_FUNC_0ARGS(GAP_SHA256_INIT), GVAR_FUNC_2ARGS(GAP_SHA256_UPDATE, state, bytes), GVAR_FUNC_1ARGS(GAP_SHA256_FINAL, state), + GVAR_FUNC_2ARGS(GAP_SHA256_FILE, filename, decompress), GVAR_FUNC_2ARGS(GAP_SHA256_HMAC, key, text), { 0 } // Finish with an empty entry diff --git a/tst/testinstall/sha256.tst b/tst/testinstall/sha256.tst index 7f6988cb3a..7350ea87ba 100644 --- a/tst/testinstall/sha256.tst +++ b/tst/testinstall/sha256.tst @@ -1,4 +1,5 @@ # +#@local dir, gzname, name, out, state, str gap> START_TEST("sha256.tst"); # @@ -57,5 +58,43 @@ gap> HexSHA256(InputTextString("abcd\r\n")); gap> HexSHA256(InputTextString("")); "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +# HexSHA256File +gap> dir := DirectoryTemporary();; +gap> name := Filename(dir, "test.txt");; +gap> FileString(name, "abcd");; +gap> HexSHA256File(name, false); +"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589" +gap> FileString(Filename(dir, "empty.txt"), "");; +gap> HexSHA256File(Filename(dir, "empty.txt"), false); +"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +# larger than the read buffer, so that more than one chunk gets hashed +gap> str := Concatenation(List([1 .. 5000], i -> "0123456789"));; +gap> FileString(Filename(dir, "big.txt"), str);; +gap> HexSHA256File(Filename(dir, "big.txt"), false) = HexSHA256(str); +true + +# a file that is not there +gap> HexSHA256File(Filename(dir, "no-such-file"), false); +fail + +# the two answers a '.gz' file has +gap> gzname := Filename(dir, "compressed.txt.gz");; +gap> out := OutputGzipFile(gzname, false);; +gap> WriteAll(out, "abcd");; +gap> CloseStream(out); +gap> HexSHA256File(gzname, false) = HexSHA256("abcd"); +false +gap> HexSHA256File(gzname, true) = HexSHA256("abcd"); +true + +# argument checking +gap> HexSHA256File(name); +Error, Function: number of arguments must be 2 (not 1) +gap> HexSHA256File(42, false); +Error, must be a string +gap> HexSHA256File(name, "yes"); +Error, must be 'true' or 'false' + # gap> STOP_TEST("sha256.tst");