diff --git a/doc/ref/string.xml b/doc/ref/string.xml index 69e815649d..42d267fc69 100644 --- a/doc/ref/string.xml +++ b/doc/ref/string.xml @@ -692,6 +692,10 @@ gap> HexStringInt(last); <#Include Label="CrcString"> <#Include Label="HexSHA256"> <#Include Label="HexSHA256File"> +<#Include Label="IsSHA256State"> +<#Include Label="SHA256State"> +<#Include Label="UpdateSHA256"> +<#Include Label="UpdateSHA256File"> <#Include Label="Pluralize"> diff --git a/lib/files.gd b/lib/files.gd index 4f5bf23ee4..df0d4e4c16 100644 --- a/lib/files.gd +++ b/lib/files.gd @@ -828,6 +828,7 @@ DeclareGlobalFunction( "Edit" ); ## ## ## +## ## ## ## hash function @@ -837,6 +838,10 @@ DeclareGlobalFunction( "Edit" ); ## (see Chapter  to learn about streams) ## when read from the current position until EOF (end-of-file). ##

+## Given a SHA-256 state (see ), return the checksum +## of everything accumulated in it so far. Reading it does not consume the +## state: it may be read again, and fed more data afterwards. +##

## The checksum is returned as string with 64 lowercase hexadecimal digits. ## HexSHA256("abcd"); @@ -889,5 +894,91 @@ DeclareGlobalFunction("HexSHA256"); ## DeclareGlobalFunction("HexSHA256File"); +## <#GAPDoc Label="IsSHA256State"> +## +## +## +## +## The category of SHA-256 states, as returned by +## . +## +## +## <#/GAPDoc> +## +DeclareCategory("IsSHA256State", IsObject); + +BIND_GLOBAL("GAP_SHA256_State_Family", NewFamily("GAP_SHA256_State_Family")); + BIND_GLOBAL("GAP_SHA256_State_Type", - NewType(NewFamily("GAP_SHA256_State_Family"), IsObject) ); + NewType(GAP_SHA256_State_Family, IsSHA256State) ); + + +## <#GAPDoc Label="SHA256State"> +## +## +## +## +## hash function +## checksum +## Return a new SHA-256 state, which accumulates data fed to it with +## and . Its digest +## is read with . +## s := SHA256State();; +## gap> UpdateSHA256(s, "ab");; +## gap> UpdateSHA256(s, "cd");; +## gap> HexSHA256(s) = HexSHA256("abcd"); +## true +## ]]> +## +## +## <#/GAPDoc> +## +DeclareGlobalFunction("SHA256State"); + + +## <#GAPDoc Label="UpdateSHA256"> +## +## +## +## +## Append string to the data accumulated in state. +## string is left untouched. +## +## +## <#/GAPDoc> +## +DeclareGlobalFunction("UpdateSHA256"); + + +## <#GAPDoc Label="UpdateSHA256File"> +## +## +## +## +## Append the contents of the file filename to the data accumulated in +## state. Return true on success, or fail if the file +## cannot be read, in which case state is left as it was. +##

+## The file is read as binary data, so the result always describes the bytes +## on disk, also on systems which distinguish text and binary mode and would +## otherwise translate line endings. It is read in chunks, so the size of +## the file is not limited by the available memory. +##

+## decompress means what it does for +## , and is likewise required. +## name := Filename(DirectoryTemporary(), "test.txt");; +## gap> FileString(name, "cd");; +## gap> s := SHA256State();; +## gap> UpdateSHA256(s, "ab"); +## gap> UpdateSHA256File(s, name, false); +## true +## gap> HexSHA256(s) = HexSHA256("abcd"); +## true +## ]]> +## +## +## <#/GAPDoc> +## +DeclareGlobalFunction("UpdateSHA256File"); diff --git a/lib/files.gi b/lib/files.gi index 90761d325e..3999fce905 100644 --- a/lib/files.gi +++ b/lib/files.gi @@ -407,10 +407,47 @@ function(words) return res; end); +InstallGlobalFunction( SHA256State, GAP_SHA256_INIT ); + +InstallMethod( PrintObj, "for a SHA256 state", [ IsSHA256State ], +function(state) + Print(""); +end); + +InstallGlobalFunction( UpdateSHA256, +function(state, string) + if not IsSHA256State(state) then + ErrorNoReturn(" must be a SHA256 state"); + elif not IsString(string) then + ErrorNoReturn(" must be a string"); + fi; + + # CopyToStringRep: the kernel converts its argument to a string in place, + # which would retype a list of characters belonging to the caller. + GAP_SHA256_UPDATE(state, CopyToStringRep(string)); +end); + +InstallGlobalFunction( UpdateSHA256File, +function(state, filename, decompress) + if not IsSHA256State(state) then + ErrorNoReturn(" must be a SHA256 state"); + elif not IsString(filename) then + ErrorNoReturn(" must be a string"); + elif not decompress in [ true, false ] then + ErrorNoReturn(" must be 'true' or 'false'"); + fi; + + return GAP_SHA256_UPDATE_FILE(state, UserHomeExpand(filename), decompress); +end); + InstallGlobalFunction( HexSHA256, function(str) local s, chunk; + if IsSHA256State(str) then + return GAP_SHA256_HexOfWords(GAP_SHA256_DIGEST(str)); + fi; + s := GAP_SHA256_INIT(); if IsString(str) then GAP_SHA256_UPDATE(s, CopyToStringRep(str)); @@ -423,25 +460,20 @@ function(str) fi; until not IsString(chunk) or Length(chunk) = 0; else - ErrorNoReturn(" has to be a string or an input stream"); + ErrorNoReturn(" has to be a string, an input stream, or a ", + "SHA256 state"); fi; - return GAP_SHA256_HexOfWords(GAP_SHA256_FINAL(s)); + return GAP_SHA256_HexOfWords(GAP_SHA256_DIGEST(s)); end); InstallGlobalFunction( HexSHA256File, function(filename, decompress) - local res; - - if not IsString(filename) then - ErrorNoReturn(" must be a string"); - elif not decompress in [ true, false ] then - ErrorNoReturn(" must be 'true' or 'false'"); - fi; + local s; - res := GAP_SHA256_FILE(UserHomeExpand(filename), decompress); - if res = fail then + s := SHA256State(); + if UpdateSHA256File(s, filename, decompress) = fail then return fail; fi; - return GAP_SHA256_HexOfWords(res); + return HexSHA256(s); end); diff --git a/src/sha256.c b/src/sha256.c index 48aa5abbee..4ea769b951 100644 --- a/src/sha256.c +++ b/src/sha256.c @@ -240,12 +240,63 @@ static int sha256_final(sha256_state_t * state) return 0; } +static sha256_state_t * SHA256_STATE(Obj state) +{ + return (sha256_state_t *)(&ADDR_OBJ(state)[1]); +} + +#define RequireSHA256State(funcname, op) \ + RequireArgumentCondition(funcname, op, \ + IS_DATOBJ(op) && \ + TYPE_OBJ(op) == GAP_SHA256_State_Type, \ + "must be a SHA256 state") + +// Feed the contents of into . Mode "rb" rather than "r" so +// that no line ending translation happens; the digest has to describe the +// bytes on disk. With set, a file whose name ends in '.gz' is +// read as its decompressed content instead, matching 'InputTextFile'. +// Returns 0 on success, -1 if the file could not be read. +static int sha256_update_file(sha256_state_t * st, + Obj filename, + Obj decompress) +{ + Int fid, len; + UChar buf[16384]; + + fid = SyFopen(CONST_CSTR_STRING(filename), "rb", decompress == True); + if (fid == -1) + return -1; + + while ((len = SyRead(fid, buf, sizeof(buf))) > 0) { + sha256_update(st, buf, len); + } + SyFclose(fid); + return len < 0 ? -1 : 0; +} + +// The eight words of as a plain list, most significant first. is +// taken by value on purpose: NEW_PLIST below may trigger a garbage collection, +// which can move bags, so this must not be handed a pointer into one. +static Obj sha256_words(sha256_state_t st) +{ + Obj result; + int i; + + result = NEW_PLIST(T_PLIST, 8); + SET_LEN_PLIST(result, 8); + for (i = 0; i < 8; i++) { + SET_ELM_PLIST(result, i + 1, ObjInt_UInt(st.r[i])); + CHANGED_BAG(result); + } + return result; +} + Obj FuncGAP_SHA256_INIT(Obj self) { Obj result; sha256_state_t * sptr; - result = NewBag(T_DATOBJ, sizeof(UInt4) + sizeof(sha256_state_t)); + result = NewBag(T_DATOBJ, sizeof(Obj) + sizeof(sha256_state_t)); SET_TYPE_OBJ(result, GAP_SHA256_State_Type); sptr = (sha256_state_t *)(&ADDR_OBJ(result)[1]); @@ -256,82 +307,50 @@ Obj FuncGAP_SHA256_INIT(Obj self) Obj FuncGAP_SHA256_UPDATE(Obj self, Obj state, Obj bytes) { - sha256_state_t * sptr; - - RequireArgumentCondition(SELF_NAME, state, - IS_DATOBJ(state) && - TYPE_OBJ(state) == GAP_SHA256_State_Type, - "must be a SHA256 state"); + RequireSHA256State(SELF_NAME, state); RequireStringRep(SELF_NAME, bytes); - sptr = (sha256_state_t *)(&ADDR_OBJ(state)[1]); - sha256_update(sptr, CHARS_STRING(bytes), GET_LEN_STRING(bytes)); + sha256_update(SHA256_STATE(state), CHARS_STRING(bytes), + GET_LEN_STRING(bytes)); CHANGED_BAG(state); return 0; } -Obj FuncGAP_SHA256_FINAL(Obj self, Obj state) +// Feed a whole file into . Returns 'true' on success, or 'fail' if +// the file could not be read, in which case is left as it was. +Obj FuncGAP_SHA256_UPDATE_FILE(Obj self, Obj state, Obj filename, + Obj decompress) { - Obj result; - sha256_state_t * sptr; - int i; + sha256_state_t st; - RequireArgumentCondition(SELF_NAME, state, - IS_DATOBJ(state) && - TYPE_OBJ(state) == GAP_SHA256_State_Type, - "must be a SHA256 state"); + RequireSHA256State(SELF_NAME, state); + RequireStringRep(SELF_NAME, filename); + RequireTrueOrFalse(SELF_NAME, decompress); - result = NEW_PLIST(T_PLIST, 8); - SET_LEN_PLIST(result, 8); + // Work on a copy, so that a file which turns out to be unreadable part + // way through does not leave half of itself in the caller's state. + st = *SHA256_STATE(state); + if (sha256_update_file(&st, filename, decompress) < 0) + return Fail; - sptr = (sha256_state_t *)(&ADDR_OBJ(state)[1]); - sha256_final(sptr); + *SHA256_STATE(state) = st; CHANGED_BAG(state); - - for (i = 0; i < 8; i++) { - SET_ELM_PLIST(result, i + 1, ObjInt_UInt(sptr->r[i])); - CHANGED_BAG(result); - } - return result; + return True; } -Obj FuncGAP_SHA256_FILE(Obj self, Obj filename, Obj decompress) +// The digest of as it stands, leaving usable. Padding a +// SHA256 state is destructive, so this finalizes a copy: reading the digest +// must not be a one-shot operation the caller has to know about. +Obj FuncGAP_SHA256_DIGEST(Obj self, Obj state) { - Obj result; sha256_state_t st; - Int fid, len; - int i; - UChar buf[16384]; - - RequireStringRep(SELF_NAME, filename); - RequireTrueOrFalse(SELF_NAME, decompress); - - // Mode "rb" rather than "r" so that no line ending translation happens; - // the digest has to describe the bytes on disk. With set, - // a file whose name ends in '.gz' is hashed as its decompressed content - // instead, matching what 'InputTextFile' would read. - fid = SyFopen(CONST_CSTR_STRING(filename), "rb", decompress == True); - if (fid == -1) - return Fail; - sha256_init(&st); - while ((len = SyRead(fid, buf, sizeof(buf))) > 0) { - sha256_update(&st, buf, len); - } - SyFclose(fid); - if (len < 0) - return Fail; + RequireSHA256State(SELF_NAME, state); + st = *SHA256_STATE(state); sha256_final(&st); - - result = NEW_PLIST(T_PLIST, 8); - SET_LEN_PLIST(result, 8); - for (i = 0; i < 8; i++) { - SET_ELM_PLIST(result, i + 1, ObjInt_UInt(st.r[i])); - CHANGED_BAG(result); - } - return result; + return sha256_words(st); } Obj FuncGAP_SHA256_HMAC(Obj self, Obj key, Obj text) @@ -393,8 +412,8 @@ Obj FuncGAP_SHA256_HMAC(Obj self, Obj key, Obj text) static StructGVarFunc GVarFuncs[] = { GVAR_FUNC_0ARGS(GAP_SHA256_INIT), GVAR_FUNC_2ARGS(GAP_SHA256_UPDATE, state, bytes), - GVAR_FUNC_1ARGS(GAP_SHA256_FINAL, state), - GVAR_FUNC_2ARGS(GAP_SHA256_FILE, filename, decompress), + GVAR_FUNC_3ARGS(GAP_SHA256_UPDATE_FILE, state, filename, decompress), + GVAR_FUNC_1ARGS(GAP_SHA256_DIGEST, state), GVAR_FUNC_2ARGS(GAP_SHA256_HMAC, key, text), { 0 } // Finish with an empty entry diff --git a/tst/testinstall/sha256.tst b/tst/testinstall/sha256.tst index 7350ea87ba..b176ccab9f 100644 --- a/tst/testinstall/sha256.tst +++ b/tst/testinstall/sha256.tst @@ -1,12 +1,12 @@ # -#@local dir, gzname, name, out, state, str +#@local c, dir, gzname, l, name, out, s, state, str, t gap> START_TEST("sha256.tst"); # # test input validation for the kernel functions # gap> state := GAP_SHA256_INIT(); - + # gap> GAP_SHA256_UPDATE(fail, fail); @@ -96,5 +96,97 @@ Error, must be a string gap> HexSHA256File(name, "yes"); Error, must be 'true' or 'false' +# +# SHA256State: accumulating input that is not in one place +# +gap> s := SHA256State(); + +gap> IsSHA256State(s); +true +gap> IsSHA256State(fail); +false + +# How the input is divided up between calls makes no difference. +gap> UpdateSHA256(s, "ab"); +gap> UpdateSHA256(s, "cd"); +gap> HexSHA256(s) = HexSHA256("abcd"); +true +gap> t := SHA256State();; +gap> for c in "abcd" do UpdateSHA256(t, [c]); od; +gap> HexSHA256(t) = HexSHA256("abcd"); +true +gap> HexSHA256(SHA256State()) = HexSHA256(""); +true + +# Reading the digest does not consume the state. Finalizing a SHA256 state +# pads it in place and so destroys it; GAP_SHA256_DIGEST does that to a copy, +# which is what makes reading a digest an ordinary operation. +gap> HexSHA256(s) = HexSHA256(s); +true +gap> UpdateSHA256(s, "e"); +gap> HexSHA256(s) = HexSHA256("abcde"); +true + +# The argument is left alone: the kernel converts a list of characters to a +# string in place, which must not happen to the caller's list. +gap> l := List("ab", c -> c);; +gap> UpdateSHA256(SHA256State(), l); +gap> IsStringRep(l); +false + +# +# UpdateSHA256File +# +gap> dir := DirectoryTemporary();; +gap> name := Filename(dir, "half.txt");; +gap> FileString(name, "cd");; +gap> s := SHA256State();; +gap> UpdateSHA256(s, "ab"); +gap> UpdateSHA256File(s, name, false); +true +gap> HexSHA256(s) = HexSHA256("abcd"); +true + +# A file that cannot be read leaves the state as it was. +gap> UpdateSHA256File(s, Filename(dir, "no-such-file"), false); +fail +gap> HexSHA256(s) = HexSHA256("abcd"); +true + +# ... and the two answers a '.gz' file has, as for HexSHA256File. +gap> gzname := Filename(dir, "c.txt.gz");; +gap> out := OutputGzipFile(gzname, false);; +gap> WriteAll(out, "abcd");; +gap> CloseStream(out); +gap> s := SHA256State();; UpdateSHA256File(s, gzname, true);; +gap> HexSHA256(s) = HexSHA256("abcd"); +true +gap> s := SHA256State();; UpdateSHA256File(s, gzname, false);; +gap> HexSHA256(s) = HexSHA256File(gzname, false); +true + +# What this is for: a git object is a header followed by a body, and neither +# has to be brought together in memory to be hashed. +gap> FileString(name, "hello artifact\n");; +gap> s := SHA256State();; +gap> UpdateSHA256(s, "blob 15\000"); +gap> UpdateSHA256File(s, name, false); +true +gap> HexSHA256(s); +"68a5de2c96dafbd696ef65b07482b6acdc0de5b763d5e49006638cab5ae0542f" + +# ... which is what 'git hash-object' reports for the same file in a +# repository created with 'git init --object-format=sha256'. + +# argument checking +gap> UpdateSHA256(fail, "a"); +Error, must be a SHA256 state +gap> UpdateSHA256(SHA256State(), 42); +Error, must be a string +gap> UpdateSHA256File(SHA256State(), name, "yes"); +Error, must be 'true' or 'false' +gap> HexSHA256(42); +Error, has to be a string, an input stream, or a SHA256 state + # gap> STOP_TEST("sha256.tst");