Mercurial > hg > nimfks

comparison src/matlab/DataHash.m @ 0:c52bc3e8d3ad tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
user: boblsturm branch 'default' added README.md added assets/.DS_Store added assets/playButton.jpg added assets/stopButton.png added assets/swapButton.jpg added data/.DS_Store added data/fiveoctaves.mp3 added data/glock2.wav added data/sinScale.mp3 added data/speech_female.mp3 added data/sweep.wav added nimfks.m.lnk added src/.DS_Store added src/matlab/.DS_Store added src/matlab/AnalysisCache.m added src/matlab/CSS.m added src/matlab/DataHash.m added src/matlab/ExistsInCache.m added src/matlab/KLDivCost.m added src/matlab/LoadFromCache.m added src/matlab/SA_B_NMF.m added src/matlab/SaveInCache.m added src/matlab/Sound.m added src/matlab/SynthesisCache.m added src/matlab/chromagram_E.m added src/matlab/chromagram_IF.m added src/matlab/chromagram_P.m added src/matlab/chromsynth.m added src/matlab/computeSTFTFeat.m added src/matlab/controller.m added src/matlab/decibelSliderReleaseCallback.m added src/matlab/drawClickCallBack.m added src/matlab/fft2chromamx.m added src/matlab/hz2octs.m added src/matlab/ifgram.m added src/matlab/ifptrack.m added src/matlab/istft.m added src/matlab/nimfks.fig added src/matlab/nimfks.m added src/matlab/nmfFn.m added src/matlab/nmf_beta.m added src/matlab/nmf_divergence.m added src/matlab/nmf_euclidean.m added src/matlab/prune_corpus.m added src/matlab/rot_kernel.m added src/matlab/templateAdditionResynth.m added src/matlab/templateDelCb.m added src/matlab/templateScrollCb.m
author boblsturm
date Sun, 18 Jun 2017 06:26:13 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:c52bc3e8d3ad
1 function Hash = DataHash(Data, Opt)
2 % DATAHASH - Checksum for Matlab array of any type
3 % This function creates a hash value for an input of any type. The type and
4 % dimensions of the input are considered as default, such that UINT8([0,0]) and
5 % UINT16(0) have different hash values. Nested STRUCTs and CELLs are parsed
6 % recursively.
7 %
8 % Hash = DataHash(Data, Opt)
9 % INPUT:
10 % Data: Array of these built-in types:
11 % (U)INT8/16/32/64, SINGLE, DOUBLE, (real/complex, full/sparse)
12 % CHAR, LOGICAL, CELL (nested), STRUCT (scalar or array, nested),
13 % function_handle.
14 % Opt: Struct to specify the hashing algorithm and the output format.
15 % Opt and all its fields are optional.
16 % Opt.Method: String, known methods for Java 1.6 (Matlab 2011b):
17 % 'SHA-1', 'SHA-256', 'SHA-384', 'SHA-512', 'MD2', 'MD5'.
18 % Call DataHash without inputs to get a list of available methods.
19 % Default: 'MD5'.
20 % Opt.Format: String specifying the output format:
21 % 'hex', 'HEX': Lower/uppercase hexadecimal string.
22 % 'double', 'uint8': Numerical vector.
23 % 'base64': Base64 encoded string, only printable ASCII
24 % characters, shorter than 'hex', no padding.
25 % Default: 'hex'.
26 % Opt.Input: Type of the input as string, not case-sensitive:
27 % 'array': The contents, type and size of the input [Data] are
28 % considered for the creation of the hash. Nested CELLs
29 % and STRUCT arrays are parsed recursively. Empty arrays of
30 % different type reply different hashs.
31 % 'file': [Data] is treated as file name and the hash is calculated
32 % for the files contents.
33 % 'bin': [Data] is a numerical, LOGICAL or CHAR array. Only the
34 % binary contents of the array is considered, such that
35 % e.g. empty arrays of different type reply the same hash.
36 % 'ascii': Same as 'bin', but only the 8-bit ASCII part of the 16-bit
37 % Matlab CHARs is considered.
38 % Default: 'array'.
39 %
40 % OUTPUT:
41 % Hash: String, DOUBLE or UINT8 vector. The length depends on the hashing
42 % method.
43 %
44 % EXAMPLES:
45 % % Default: MD5, hex:
46 % DataHash([]) % 5b302b7b2099a97ba2a276640a192485
47 % % MD5, Base64:
48 % Opt = struct('Format', 'base64', 'Method', 'MD5');
49 % DataHash(int32(1:10), Opt) % +tJN9yeF89h3jOFNN55XLg
50 % % SHA-1, Base64:
51 % S.a = uint8([]);
52 % S.b = {{1:10}, struct('q', uint64(415))};
53 % Opt.Method = 'SHA-1';
54 % Opt.Format = 'HEX';
55 % DataHash(S, Opt) % 18672BE876463B25214CA9241B3C79CC926F3093
56 % % SHA-1 of binary values:
57 % Opt = struct('Method', 'SHA-1', 'Input', 'bin');
58 % DataHash(1:8, Opt) % 826cf9d3a5d74bbe415e97d4cecf03f445f69225
59 % % SHA-256, consider ASCII part only (Matlab's CHAR has 16 bits!):
60 % Opt.Method = 'SHA-256';
61 % Opt.Input = 'ascii';
62 % DataHash('abc', Opt)
63 % % ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad
64 % % Or equivalently:
65 % Opt.Input = 'bin';
66 % DataHash(uint8('abc'), Opt)
67 %
68 % NOTES:
69 % Function handles and user-defined objects cannot be converted uniquely:
70 % - The subfunction ConvertFuncHandle uses the built-in function FUNCTIONS,
71 % but the replied struct can depend on the Matlab version.
72 % - It is tried to convert objects to UINT8 streams in the subfunction
73 % ConvertObject. A conversion by STRUCT() might be more appropriate.
74 % Adjust these subfunctions on demand.
75 %
76 % MATLAB CHARs have 16 bits! Use Opt.Input='ascii' for comparisons with e.g.
77 % online hash generators.
78 %
79 % Matt Raum suggested this for e.g. user-defined objects:
80 % DataHash(getByteStreamFromArray(Data)
81 % This works very well, but unfortunately getByteStreamFromArray is
82 % undocumented, such that it might vanish in the future or reply different
83 % output.
84 %
85 % For arrays the calculated hash value might be changed in new versions.
86 % Calling this function without inputs replies the version of the hash.
87 %
88 % The C-Mex function GetMD5 is 2 to 100 times faster, but obtains MD5 only:
89 % http://www.mathworks.com/matlabcentral/fileexchange/25921
90 %
91 % Tested: Matlab 7.7, 7.8, 7.13, 8.6, WinXP/32, Win7/64
92 % Author: Jan Simon, Heidelberg, (C) 2011-2016 matlab.2010(a)n(MINUS)simon.de
93 %
94 % See also: TYPECAST, CAST.
95 %
96 % Michael Kleder, "Compute Hash", no structs and cells:
97 % http://www.mathworks.com/matlabcentral/fileexchange/8944
98 % Tim, "Serialize/Deserialize", converts structs and cells to a byte stream:
99 % http://www.mathworks.com/matlabcentral/fileexchange/29457
100
101 % $JRev: R-H V:033 Sum:R+m7rAPNLvlw Date:18-Jun-2016 14:33:17 $
102 % $License: BSD (use/copy/change/redistribute on own risk, mention the author) $
103 % $File: Tools\GLFile\DataHash.m $
104 % History:
105 % 001: 01-May-2011 21:52, First version.
106 % 007: 10-Jun-2011 10:38, [Opt.Input], binary data, complex values considered.
107 % 011: 26-May-2012 15:57, Fixed: Failed for binary input and empty data.
108 % 014: 04-Nov-2012 11:37, Consider Mex-, MDL- and P-files also.
109 % Thanks to David (author 243360), who found this bug.
110 % Jan Achterhold (author 267816) suggested to consider Java objects.
111 % 016: 01-Feb-2015 20:53, Java heap space exhausted for large files.
112 % Now files are process in chunks to save memory.
113 % 017: 15-Feb-2015 19:40, Collsions: Same hash for different data.
114 % Examples: zeros(1,1) and zeros(1,1,0)
115 % complex(0) and zeros(1,1,0,0)
116 % Now the number of dimensions is included, to avoid this.
117 % 022: 30-Mar-2015 00:04, Bugfix: Failed for strings and [] without TYPECASTX.
118 % Ross found these 2 bugs, which occur when TYPECASTX is not installed.
119 % If you need the base64 format padded with '=' characters, adjust
120 % fBase64_enc as you like.
121 % 026: 29-Jun-2015 00:13, Changed hash for STRUCTs.
122 % Struct arrays are analysed field by field now, which is much faster.
123 % 027: 13-Sep-2015 19:03, 'ascii' input as abbrev. for Input='bin' and UINT8().
124 % 028: 15-Oct-2015 23:11, Example values in help section updated to v022.
125 % 029: 16-Oct-2015 22:32, Use default options for empty input.
126 % 031: 28-Feb-2016 15:10, New hash value to get same reply as GetMD5.
127 % New Matlab version (at least 2015b) use a fast method for TYPECAST, such
128 % that calling James Tursa's TYPECASTX is not needed anymore.
129 % Matlab 6.5 not supported anymore: MException for CATCH.
130 % 033: 18-Jun-2016 14:28, BUGFIX: Failed on empty files.
131 % Thanks to Christian (AuthorID 2918599).
132
133 % OPEN BUGS:
134 % Nath wrote:
135 % function handle refering to struct containing the function will create
136 % infinite loop. Is there any workaround ?
137 % Example:
138 % d= dynamicprops();
139 % addprop(d,'f');
140 % d.f= @(varargin) struct2cell(d);
141 % DataHash(d.f) % infinite loop
142 % This is caught with an error message concerning the recursion limit now.
143
144 % Main function: ===============================================================
145 % Default options: -------------------------------------------------------------
146 Method = 'MD5';
147 OutFormat = 'hex';
148 isFile = false;
149 isBin = false;
150
151 % Check number and type of inputs: ---------------------------------------------
152 nArg = nargin;
153 if nArg == 2
154 if isa(Opt, 'struct') == 0 % Bad type of 2nd input:
155 Error_L('BadInput2', '2nd input [Opt] must be a struct.');
156 end
157
158 % Specify hash algorithm:
159 if isfield(Opt, 'Method') && ~isempty(Opt.Method) % Short-circuiting
160 Method = upper(Opt.Method);
161 end
162
163 % Specify output format:
164 if isfield(Opt, 'Format') && ~isempty(Opt.Format) % Short-circuiting
165 OutFormat = Opt.Format;
166 end
167
168 % Check if the Input type is specified - default: 'array':
169 if isfield(Opt, 'Input') && ~isempty(Opt.Input) % Short-circuiting
170 if strcmpi(Opt.Input, 'File')
171 if ischar(Data) == 0
172 Error_L('CannotOpen', '1st input FileName must be a string');
173 end
174 isFile = true;
175
176 elseif strncmpi(Opt.Input, 'bin', 3) % Accept 'binary' also
177 if (isnumeric(Data) || ischar(Data) || islogical(Data)) == 0 || ...
178 issparse(Data)
179 Error_L('BadDataType', ...
180 '1st input must be numeric, CHAR or LOGICAL for binary input.');
181 end
182 isBin = true;
183
184 elseif strncmpi(Opt.Input, 'asc', 3) % 8-bit ASCII characters
185 if ~ischar(Data)
186 Error_L('BadDataType', ...
187 '1st input must be a CHAR for the input type ASCII.');
188 end
189 isBin = true;
190 Data = uint8(Data);
191 end
192 end
193
194 elseif nArg == 0 % Reply version of this function:
195 R = Version_L;
196
197 if nargout == 0
198 disp(R);
199 else
200 Hash = R;
201 end
202
203 return;
204
205 elseif nArg ~= 1 % Bad number of arguments:
206 Error_L('BadNInput', '1 or 2 inputs required.');
207 end
208
209 % Create the engine: -----------------------------------------------------------
210 try
211 Engine = java.security.MessageDigest.getInstance(Method);
212 catch
213 Error_L('BadInput2', 'Invalid algorithm: [%s].', Method);
214 end
215
216 % Create the hash value: -------------------------------------------------------
217 if isFile
218 % Open the file:
219 FID = fopen(Data, 'r');
220 if FID < 0
221 % Check existence of file:
222 Found = FileExist_L(Data);
223 if Found
224 Error_L('CantOpenFile', 'Cannot open file: %s.', Data);
225 else
226 Error_L('FileNotFound', 'File not found: %s.', Data);
227 end
228 end
229
230 % Read file in chunks to save memory and Java heap space:
231 Chunk = 1e6; % Fastest for 1e6 on Win7/64, HDD
232 Count = Chunk; % Dummy value to satisfy WHILE condition
233 while Count == Chunk
234 [Data, Count] = fread(FID, Chunk, '*uint8');
235 if Count ~= 0 % Avoid error for empty file
236 Engine.update(Data);
237 end
238 end
239 fclose(FID);
240
241 % Calculate the hash:
242 Hash = typecast(Engine.digest, 'uint8');
243
244 elseif isBin % Contents of an elementary array, type tested already:
245 if isempty(Data) % Nothing to do, Engine.update fails for empty input!
246 Hash = typecast(Engine.digest, 'uint8');
247 else % Matlab's TYPECAST is less elegant:
248 if isnumeric(Data)
249 if isreal(Data)
250 Engine.update(typecast(Data(:), 'uint8'));
251 else
252 Engine.update(typecast(real(Data(:)), 'uint8'));
253 Engine.update(typecast(imag(Data(:)), 'uint8'));
254 end
255 elseif islogical(Data) % TYPECAST cannot handle LOGICAL
256 Engine.update(typecast(uint8(Data(:)), 'uint8'));
257 elseif ischar(Data) % TYPECAST cannot handle CHAR
258 Engine.update(typecast(uint16(Data(:)), 'uint8'));
259 % Bugfix: Line removed
260 end
261 Hash = typecast(Engine.digest, 'uint8');
262 end
263 else % Array with type:
264 Engine = CoreHash(Data, Engine);
265 Hash = typecast(Engine.digest, 'uint8');
266 end
267
268 % Convert hash specific output format: -----------------------------------------
269 switch OutFormat
270 case 'hex'
271 Hash = sprintf('%.2x', double(Hash));
272 case 'HEX'
273 Hash = sprintf('%.2X', double(Hash));
274 case 'double'
275 Hash = double(reshape(Hash, 1, []));
276 case 'uint8'
277 Hash = reshape(Hash, 1, []);
278 case 'base64'
279 Hash = fBase64_enc(double(Hash));
280 otherwise
281 Error_L('BadOutFormat', ...
282 '[Opt.Format] must be: HEX, hex, uint8, double, base64.');
283 end
284
285 % return;
286
287 % ******************************************************************************
288 function Engine = CoreHash(Data, Engine)
289 % This methods uses the slower TYPECAST of Matlab
290
291 % Consider the type and dimensions of the array to distinguish arrays with the
292 % same data, but different shape: [0 x 0] and [0 x 1], [1,2] and [1;2],
293 % DOUBLE(0) and SINGLE([0,0]):
294 % < v016: [class, size, data]. BUG! 0 and zeros(1,1,0) had the same hash!
295 % >= v016: [class, ndims, size, data]
296 Engine.update([uint8(class(Data)), ...
297 typecast(uint64([ndims(Data), size(Data)]), 'uint8')]);
298
299 if issparse(Data) % Sparse arrays to struct:
300 [S.Index1, S.Index2, S.Value] = find(Data);
301 Engine = CoreHash(S, Engine);
302 elseif isstruct(Data) % Hash for all array elements and fields:
303 F = sort(fieldnames(Data)); % Ignore order of fields
304 for iField = 1:length(F) % Loop over fields
305 aField = F{iField};
306 Engine.update(uint8(aField));
307 for iS = 1:numel(Data) % Loop over elements of struct array
308 Engine = CoreHash(Data(iS).(aField), Engine);
309 end
310 end
311 elseif iscell(Data) % Get hash for all cell elements:
312 for iS = 1:numel(Data)
313 Engine = CoreHash(Data{iS}, Engine);
314 end
315 elseif isempty(Data) % Nothing to do
316 elseif isnumeric(Data)
317 if isreal(Data)
318 Engine.update(typecast(Data(:), 'uint8'));
319 else
320 Engine.update(typecast(real(Data(:)), 'uint8'));
321 Engine.update(typecast(imag(Data(:)), 'uint8'));
322 end
323 elseif islogical(Data) % TYPECAST cannot handle LOGICAL
324 Engine.update(typecast(uint8(Data(:)), 'uint8'));
325 elseif ischar(Data) % TYPECAST cannot handle CHAR
326 Engine.update(typecast(uint16(Data(:)), 'uint8'));
327 elseif isa(Data, 'function_handle')
328 Engine = CoreHash(ConvertFuncHandle(Data), Engine);
329 elseif (isobject(Data) || isjava(Data)) && ismethod(Data, 'hashCode')
330 Engine = CoreHash(char(Data.hashCode), Engine);
331 else % Most likely a user-defined object:
332 try
333 BasicData = ConvertObject(Data);
334 catch ME
335 error(['JSimon:', mfilename, ':BadDataType'], ...
336 '%s: Cannot create elementary array for type: %s\n %s', ...
337 mfilename, class(Data), ME.message);
338 end
339
340 try
341 Engine = CoreHash(BasicData, Engine);
342 catch ME
343 if strcmpi(ME.identifier, 'MATLAB:recursionLimit')
344 ME = MException(['JSimon:', mfilename, ':RecursiveType'], ...
345 '%s: Cannot create hash for recursive data type: %s', ...
346 mfilename, class(Data));
347 end
348 throw(ME);
349 end
350 end
351
352 % return;
353
354 % ******************************************************************************
355 function FuncKey = ConvertFuncHandle(FuncH)
356 % The subfunction ConvertFuncHandle converts function_handles to a struct
357 % using the Matlab function FUNCTIONS. The output of this function changes
358 % with the Matlab version, such that DataHash(@sin) replies different hashes
359 % under Matlab 6.5 and 2009a.
360 % An alternative is using the function name and name of the file for
361 % function_handles, but this is not unique for nested or anonymous functions.
362 % If the MATLABROOT is removed from the file's path, at least the hash of
363 % Matlab's toolbox functions is (usually!) not influenced by the version.
364 % Finally I'm in doubt if there is a unique method to hash function handles.
365 % Please adjust the subfunction ConvertFuncHandles to your needs.
366
367 % The Matlab version influences the conversion by FUNCTIONS:
368 % 1. The format of the struct replied FUNCTIONS is not fixed,
369 % 2. The full paths of toolbox function e.g. for @mean differ.
370 FuncKey = functions(FuncH);
371
372 % Include modification file time and file size. Suggested by Aslak Grinsted:
373 if ~isempty(FuncKey.file)
374 d = dir(FuncKey.file);
375 if ~isempty(d)
376 FuncKey.filebytes = d.bytes;
377 FuncKey.filedate = d.datenum;
378 end
379 end
380
381 % ALTERNATIVE: Use name and path. The <matlabroot> part of the toolbox functions
382 % is replaced such that the hash for @mean does not depend on the Matlab
383 % version.
384 % Drawbacks: Anonymous functions, nested functions...
385 % funcStruct = functions(FuncH);
386 % funcfile = strrep(funcStruct.file, matlabroot, '<MATLAB>');
387 % FuncKey = uint8([funcStruct.function, ' ', funcfile]);
388
389 % Finally I'm afraid there is no unique method to get a hash for a function
390 % handle. Please adjust this conversion to your needs.
391
392 % return;
393
394 % ******************************************************************************
395 function DataBin = ConvertObject(DataObj)
396 % Convert a user-defined object to a binary stream. There cannot be a unique
397 % solution, so this part is left for the user...
398
399 try % Perhaps a direct conversion is implemented:
400 DataBin = uint8(DataObj);
401
402 % Matt Raum had this excellent idea - unfortunately this function is
403 % undocumented and might not be supported in te future:
404 % DataBin = getByteStreamFromArray(DataObj);
405
406 catch % Or perhaps this is better:
407 WarnS = warning('off', 'MATLAB:structOnObject');
408 DataBin = struct(DataObj);
409 warning(WarnS);
410 end
411
412 % return;
413
414 % ******************************************************************************
415 function Out = fBase64_enc(In)
416 % Encode numeric vector of UINT8 values to base64 string.
417 % The intention of this is to create a shorter hash than the HEX format.
418 % Therefore a padding with '=' characters is omitted on purpose.
419
420 Pool = [65:90, 97:122, 48:57, 43, 47]; % [0:9, a:z, A:Z, +, /]
421 v8 = [128; 64; 32; 16; 8; 4; 2; 1];
422 v6 = [32, 16, 8, 4, 2, 1];
423
424 In = reshape(In, 1, []);
425 X = rem(floor(In(ones(8, 1), :) ./ v8(:, ones(length(In), 1))), 2);
426 Y = reshape([X(:); zeros(6 - rem(numel(X), 6), 1)], 6, []);
427 Out = char(Pool(1 + v6 * Y));
428
429 % return;
430
431 % ******************************************************************************
432 function Ex = FileExist_L(FileName)
433 % A more reliable version of EXIST(FileName, 'file'):
434 dirFile = dir(FileName);
435 if length(dirFile) == 1
436 Ex = ~(dirFile.isdir);
437 else
438 Ex = false;
439 end
440
441 % return;
442
443 % ******************************************************************************
444 function R = Version_L()
445 % The output differs between versions of this function. So give the user a
446 % chance to recognize the version:
447 % 1: 01-May-2011, Initial version
448 % 2: 15-Feb-2015, The number of dimensions is considered in addition.
449 % In version 1 these variables had the same hash:
450 % zeros(1,1) and zeros(1,1,0), complex(0) and zeros(1,1,0,0)
451 % 3: 29-Jun-2015, Struct arrays are processed field by field and not element
452 % by element, because this is much faster. In consequence the hash value
453 % differs, if the input contains a struct.
454 % 4: 28-Feb-2016 15:20, same output as GetMD5 for MD5 sums. Therefore the
455 % dimensions are casted to UINT64 at first.
456 R.HashVersion = 4;
457 R.Date = [2016, 2, 28];
458
459 R.HashMethod = {};
460 try
461 Provider = java.security.Security.getProviders;
462 for iProvider = 1:numel(Provider)
463 S = char(Provider(iProvider).getServices);
464 Index = strfind(S, 'MessageDigest.');
465 for iDigest = 1:length(Index)
466 Digest = strtok(S(Index(iDigest):end));
467 Digest = strrep(Digest, 'MessageDigest.', '');
468 R.HashMethod = cat(2, R.HashMethod, {Digest});
469 end
470 end
471 catch ME
472 fprintf(2, '%s\n', ME.message);
473 R.HashMethod = 'error';
474 end
475
476 % return;
477
478 % ******************************************************************************
479 function Error_L(ID, varargin)
480
481 error(['JSimon:', mfilename, ':', ID], ['*** %s: ', varargin{1}], ...
482 mfilename, varargin{2:nargin - 1});
483
484 % return;