Skip to content

Commit

Permalink
recovered helper files
Browse files Browse the repository at this point in the history
  • Loading branch information
samirchowdhury committed Dec 18, 2021
1 parent df5cff8 commit c8a98f0
Show file tree
Hide file tree
Showing 3 changed files with 155 additions and 0 deletions.
88 changes: 88 additions & 0 deletions code/tools/constructNPYheader.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@



function header = constructNPYheader(dataType, shape, varargin)

if ~isempty(varargin)
fortranOrder = varargin{1}; % must be true/false
littleEndian = varargin{2}; % must be true/false
else
fortranOrder = true;
littleEndian = true;
end

dtypesMatlab = {'uint8','uint16','uint32','uint64','int8','int16','int32','int64','single','double', 'logical'};
dtypesNPY = {'u1', 'u2', 'u4', 'u8', 'i1', 'i2', 'i4', 'i8', 'f4', 'f8', 'b1'};

magicString = uint8([147 78 85 77 80 89]); %x93NUMPY

majorVersion = uint8(1);
minorVersion = uint8(0);

% build the dict specifying data type, array order, endianness, and
% shape
dictString = '{''descr'': ''';

if littleEndian
dictString = [dictString '<'];
else
dictString = [dictString '>'];
end

dictString = [dictString dtypesNPY{strcmp(dtypesMatlab,dataType)} ''', '];

dictString = [dictString '''fortran_order'': '];

if fortranOrder
dictString = [dictString 'True, '];
else
dictString = [dictString 'False, '];
end

dictString = [dictString '''shape'': ('];

% if length(shape)==1 && shape==1
%
% else
% for s = 1:length(shape)
% if s==length(shape) && shape(s)==1
%
% else
% dictString = [dictString num2str(shape(s))];
% if length(shape)>1 && s+1==length(shape) && shape(s+1)==1
% dictString = [dictString ','];
% elseif length(shape)>1 && s<length(shape)
% dictString = [dictString ', '];
% end
% end
% end
% if length(shape)==1
% dictString = [dictString ','];
% end
% end

for s = 1:length(shape)
dictString = [dictString num2str(shape(s))];
if s<length(shape)
dictString = [dictString ', '];
end
end

dictString = [dictString '), '];

dictString = [dictString '}'];

totalHeaderLength = length(dictString)+10; % 10 is length of magicString, version, and headerLength

headerLengthPadded = ceil(double(totalHeaderLength+1)/16)*16; % the whole thing should be a multiple of 16
% I add 1 to the length in order to allow for the newline character

% format specification is that headerlen is little endian. I believe it comes out so using this command...
headerLength = typecast(int16(headerLengthPadded-10), 'uint8');

zeroPad = zeros(1,headerLengthPadded-totalHeaderLength, 'uint8')+uint8(32); % +32 so they are spaces
zeroPad(end) = uint8(10); % newline character

header = uint8([magicString majorVersion minorVersion headerLength dictString zeroPad]);

end
42 changes: 42 additions & 0 deletions code/tools/datToNPY.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@


function datToNPY(inFilename, outFilename, dataType, shape, varargin)
% function datToNPY(inFilename, outFilename, shape, dataType, [fortranOrder, littleEndian])
%
% make a NPY file from a flat binary file, given that you know the shape,
% dataType, ordering, and endianness of the flat binary file.
%
% The point here is you don't want to read in all the data from the
% existing binary file - instead you can just create the appropriate header
% and then concatenate it with the data.
%
% ** completely untested

if ~isempty(varargin)
fortranOrder = varargin{1}; % must be true/false
littleEndian = varargin{2}; % must be true/false
else
fortranOrder = true;
littleEndian = true;
end

header = constructNPYheader(dataType, shape, fortranOrder, littleEndian);

% ** TODO: need to put the header into a temp file instead, in case the
% outFilename is the same as the inFilename (and then delete the temp file
% later)
fid = fopen(tempFilename, 'w');
fwrite(fid, header, 'uint8');
fclose(fid)

str = computer;
switch str
case {'PCWIN', 'PCWIN64'}
[~,~] = system(sprintf('copy /b %s+%s %s', tempFilename, inFilename, outFilename));
case {'GLNXA64', 'MACI64'}
[~,~] = system(sprintf('cat %s %s > %s', tempFilename, inFilename, outFilename));

otherwise
fprintf(1, 'I don''t know how to concatenate files for your OS, but you can finish making the NPY youself by concatenating %s with %s.\n', tempFilename, inFilename);
end

25 changes: 25 additions & 0 deletions code/tools/writeNPY.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@


function writeNPY(var, filename)
% function writeNPY(var, filename)
%
% Only writes little endian, fortran (column-major) ordering; only writes
% with NPY version number 1.0.
%
% Always outputs a shape according to matlab's convention, e.g. (10, 1)
% rather than (10,).


shape = size(var);
dataType = class(var);

header = constructNPYheader(dataType, shape);

fid = fopen(filename, 'w');
fwrite(fid, header, 'uint8');
fwrite(fid, var, dataType);
fclose(fid);


end

0 comments on commit c8a98f0

Please sign in to comment.