Skip to content

Commit

Permalink
zip: Add skip_directories option
Browse files Browse the repository at this point in the history
  • Loading branch information
@garazdawi
garazdawi committed Jun 3, 2024
1 parent 6728877 commit 193339a
Show file tree
Hide file tree
Showing 2 changed files with 102 additions and 12 deletions.
59 changes: 48 additions & 11 deletions lib/stdlib/src/zip.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ convention, add `.zip` to the filename.
open_opts, % options passed to file:open
feedback, % feeback (fun)
cwd, % directory to relate paths to
skip_dirs, % skip creating empty directories
extra % The extra fields to include
}).

Expand All @@ -110,13 +111,15 @@ convention, add `.zip` to the filename.
input, % input object (fun)
raw_iterator,% applied to each dir entry
open_opts, % options passed to file:open
skip_dirs, % skip creating empty directories
extra % The extra fields to include
}).

-record(openzip_opts, {
output, % output object (fun)
open_opts, % file:open options
cwd, % directory to relate paths to
skip_dirs, % skip creating empty directories
extra % The extra fields to include
}).

Expand All @@ -129,6 +132,7 @@ convention, add `.zip` to the filename.
output, % output io object (fun)
zlib, % handle to open zlib
cwd, % directory to relate paths to
skip_dirs, % skip creating empty directories
extra % The extra fields to include
}).

Expand Down Expand Up @@ -366,6 +370,10 @@ Options:
with option `memory` specified, which means that no files are overwritten,
existing files are excluded from the result.
- **`skip_directories`** - By default empty directories within zip archives are
extracted. With option `skip_directories` set, empty directories are no longer
created.
- **`{extra, Extras}`** - The zip "extra" features to respect. The supported
"extra" features are "extended timestamps" and "UID and GID" handling.
By default only "extended timestamps" is enabled when unzipping.
Expand Down Expand Up @@ -407,7 +415,7 @@ unzip(F, Options) ->
do_unzip(F, Options) ->
Opts = get_unzip_options(F, Options),
#unzip_opts{input = Input, open_opts = OpO,
extra = ExtraOpts} = Opts,
extra = ExtraOpts} = Opts,
In0 = Input({open, F, OpO -- [write]}, []),
RawIterator = fun raw_file_info_etc/5,
{Info, In1} = get_central_dir(In0, RawIterator, Input, ExtraOpts),
Expand Down Expand Up @@ -679,6 +687,10 @@ One option is available:
Adding `cooked` to the mode list overrides the default and opens the zip file
without option `raw`.
- **`skip_directories`** - By default empty directories within zip archives are
listed. With option `skip_directories` set, empty directories are no longer
listed.
- **`{extra, Extras}`** - The zip "extra" features to respect. The supported
"extra" features are "extended timestamps" and "UID and GID" handling.
By default only "extended timestamps" is enabled when listing files.
Expand All @@ -701,11 +713,22 @@ do_list_dir(F, Options) ->
Opts = get_list_dir_options(F, Options),
#list_dir_opts{input = Input, open_opts = OpO,
raw_iterator = RawIterator,
skip_dirs = SkipDirs,
extra = ExtraOpts} = Opts,
In0 = Input({open, F, OpO}, []),
{Info, In1} = get_central_dir(In0, RawIterator, Input, ExtraOpts),
Input(close, In1),
{ok, Info}.
if SkipDirs ->
{ok,
lists:filter(
fun(#zip_file{ name = Name }) ->
lists:last(Name) =/= $/;
(#zip_comment{}) ->
true
end, Info)};
true ->
{ok, Info}
end.

-doc(#{equiv => zip_open/2}).
-spec(zip_open(Archive) -> {ok, ZipHandle} | {error, Reason} when
Expand Down Expand Up @@ -874,6 +897,8 @@ get_unzip_opt([keep_old_files | Rest], Opts) ->
Keep = fun keep_old_file/1,
Filter = fun_and_1(Keep, Opts#unzip_opts.file_filter),
get_unzip_opt(Rest, Opts#unzip_opts{file_filter = Filter});
get_unzip_opt([skip_directories | Rest], Opts) ->
get_unzip_opt(Rest, Opts#unzip_opts{skip_dirs = true});
get_unzip_opt([{extra, What} = O| Rest], Opts) when is_list(What) ->
case lists:all(fun(E) -> lists:member(E, ?EXTRA_OPTIONS) end, What) of
true ->
Expand All @@ -891,6 +916,8 @@ get_list_dir_opt([cooked | Rest], #list_dir_opts{open_opts = OpO} = Opts) ->
get_list_dir_opt([names_only | Rest], Opts) ->
get_list_dir_opt(Rest, Opts#list_dir_opts{
raw_iterator = fun(A, B, C, D, E) -> raw_name_only(A, B, C, D, E) end});
get_list_dir_opt([skip_directories | Rest], Opts) ->
get_list_dir_opt(Rest, Opts#list_dir_opts{skip_dirs = true});
get_list_dir_opt([{extra, What} = O| Rest], Opts) when is_list(What) ->
case lists:all(fun(E) -> lists:member(E, ?EXTRA_OPTIONS) end, What) of
true ->
Expand Down Expand Up @@ -1008,6 +1035,7 @@ get_unzip_options(F, Options) ->
input = get_input(F),
open_opts = [raw],
feedback = fun silent/1,
skip_dirs = false,
cwd = "",
extra = [extended_timestamp]
},
Expand All @@ -1017,6 +1045,7 @@ get_openzip_options(Options) ->
Opts = #openzip_opts{open_opts = [raw, read],
output = fun file_io/2,
cwd = "",
skip_dirs = false,
extra = ?EXTRA_OPTIONS},
get_openzip_opt(Options, Opts).

Expand Down Expand Up @@ -1046,6 +1075,7 @@ get_list_dir_options(F, Options) ->
Opts = #list_dir_opts{raw_iterator = fun raw_file_info_public/5,
input = get_input(F),
open_opts = [raw],
skip_dirs = false,
extra = [extended_timestamp]},
get_list_dir_opt(Options, Opts).

Expand Down Expand Up @@ -1664,7 +1694,7 @@ openzip_open(F, Options) ->
do_openzip_open(F, Options) ->
Opts = get_openzip_options(Options),
#openzip_opts{output = Output, open_opts = OpO, cwd = CWD,
extra = ExtraOpts} = Opts,
skip_dirs = SkipDirs, extra = ExtraOpts} = Opts,
Input = get_input(F),
In0 = Input({open, F, OpO -- [write]}, []),
{[#zip_comment{comment = C} | Files], In1} =
Expand All @@ -1677,6 +1707,7 @@ do_openzip_open(F, Options) ->
output = Output,
zlib = Z,
cwd = CWD,
skip_dirs = SkipDirs,
extra = ExtraOpts}}.

%% retrieve all files from an open archive
Expand All @@ -1687,10 +1718,12 @@ openzip_get(OpenZip) ->
end.

do_openzip_get(#openzip{files = Files, in = In0, input = Input,
output = Output, zlib = Z, cwd = CWD, extra = ExtraOpts}) ->
output = Output, zlib = Z, cwd = CWD, skip_dirs = SkipDirs,
extra = ExtraOpts}) ->
ZipOpts = #unzip_opts{output = Output, input = Input,
file_filter = fun all/1, open_opts = [],
feedback = fun silent/1, cwd = CWD, extra = ExtraOpts},
feedback = fun silent/1, cwd = CWD, skip_dirs = SkipDirs,
extra = ExtraOpts},
R = get_z_files(Files, Z, In0, ZipOpts, []),
{ok, R};
do_openzip_get(_) ->
Expand All @@ -1717,7 +1750,7 @@ do_openzip_get(F, #openzip{files = Files, in = In0, input = Input,
{#zip_file{offset = Offset},_}=ZFile ->
In1 = Input({seek, bof, Offset}, In0),
case get_z_file(In1, Z, Input, Output, [], fun silent/1,
CWD, ZFile, fun all/1, ExtraOpts) of
CWD, ZFile, fun all/1, false, ExtraOpts) of
{file, R, _In2} -> {ok, R};
_ -> throw(file_not_found)
end;
Expand Down Expand Up @@ -1825,6 +1858,8 @@ get_openzip_opt([memory | Rest], Opts) ->
get_openzip_opt(Rest, Opts#openzip_opts{output = fun binary_io/2});
get_openzip_opt([{cwd, CWD} | Rest], Opts) ->
get_openzip_opt(Rest, Opts#openzip_opts{cwd = CWD});
get_openzip_opt([skip_directories | Rest], Opts) ->
get_openzip_opt(Rest, Opts#openzip_opts{skip_dirs = true});
get_openzip_opt([{extra, What} = O| Rest], Opts) when is_list(What) ->
case lists:all(fun(E) -> lists:member(E, ?EXTRA_OPTIONS) end, What) of
true ->
Expand Down Expand Up @@ -2179,13 +2214,13 @@ get_z_files([#zip_comment{comment = _} | Rest], Z, In, Opts, Acc) ->
get_z_files([{#zip_file{offset = Offset},_} = ZFile | Rest], Z, In0,
#unzip_opts{input = Input, output = Output, open_opts = OpO,
file_filter = Filter, feedback = FB,
cwd = CWD, extra = ExtraOpts} = Opts, Acc0) ->
cwd = CWD, skip_dirs = SkipDirs, extra = ExtraOpts} = Opts, Acc0) ->
case Filter(ZFile) of
true ->
In1 = Input({seek, bof, Offset}, In0),
{In2, Acc1} =
case get_z_file(In1, Z, Input, Output, OpO, FB,
CWD, ZFile, Filter, ExtraOpts) of
CWD, ZFile, Filter, SkipDirs, ExtraOpts) of
{Type, GZD, Inx} when Type =:= file; Type =:= dir ->
{Inx, [GZD | Acc0]};
{_, Inx} -> {Inx, Acc0}
Expand All @@ -2197,7 +2232,7 @@ get_z_files([{#zip_file{offset = Offset},_} = ZFile | Rest], Z, In0,

%% get a file from the archive, reading chunks
get_z_file(In0, Z, Input, Output, OpO, FB,
CWD, {ZipFile,ZipExtra}, Filter, ExtraOpts) ->
CWD, {ZipFile,ZipExtra}, Filter, SkipDirs, ExtraOpts) ->
case Input({read, ?LOCAL_FILE_HEADER_SZ}, In0) of
{eof, In1} ->
{eof, In1};
Expand Down Expand Up @@ -2229,12 +2264,14 @@ get_z_file(In0, Z, Input, Output, OpO, FB,
{false,FileName1} ->
Filter({ZipFile#zip_file{name = FileName1},ZipExtra})
end,
case ReadAndWrite of

IsDir = lists:last(FileName) =:= $/,

case ReadAndWrite andalso not (IsDir andalso SkipDirs) of
true ->
{Type, Out, In} =
case lists:last(FileName) of
$/ ->
%% perhaps this should always be done?
Out1 = Output({ensure_path,FileName1},[]),
{dir, Out1, In3};
_ ->
Expand Down
55 changes: 54 additions & 1 deletion lib/stdlib/test/zip_SUITE.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,22 @@ unzip_options(Config) when is_list(Config) ->
lists:foreach(fun(F)-> ok = file:delete(F) end,
RetList),

%% Clean up and verify no more files.
0 = delete_files([Subdir]),

FList2 = ["abc.txt","quotes/rain.txt","wikipedia.txt","emptyFile"],

%% Unzip a zip file in Subdir
{ok, RetList2} = zip:unzip(Long, [{cwd, Subdir},skip_directories]),

%% Verify.
true = (length(RetList2) =:= 4),
lists:foreach(fun(F)-> {ok,B} = file:read_file(filename:join(DataDir, F)),
{ok,B} = file:read_file(filename:join(Subdir, F)) end,
FList2),
lists:foreach(fun(F)-> ok = file:delete(F) end,
RetList2),

%% Clean up and verify no more files.
0 = delete_files([Subdir]),
ok.
Expand Down Expand Up @@ -529,6 +545,22 @@ zip_options(Config) when is_list(Config) ->

%% Test the options for list_dir... one day.
list_dir_options(Config) when is_list(Config) ->

DataDir = get_value(data_dir, Config),
Archive = filename:join(DataDir, "abc.zip"),

{ok,
["abc.txt", "quotes/rain.txt", "empty/", "wikipedia.txt", "emptyFile" ]} =
zip:list_dir(Archive,[names_only]),

{ok,
[#zip_comment{},
#zip_file{ name = "abc.txt" },
#zip_file{ name = "quotes/rain.txt" },
#zip_file{ name = "wikipedia.txt" },
#zip_file{ name = "emptyFile" }
]} = zip:list_dir(Archive,[skip_directories]),

ok.

%% convert zip_info as returned from list_dir to a list of names
Expand Down Expand Up @@ -701,8 +733,9 @@ unzip_from_binary(Config) when is_list(Config) ->
DataDir = get_value(data_dir, Config),
PrivDir = get_value(priv_dir, Config),
ExtractDir = filename:join(PrivDir, "extract_from_binary"),
ok = file:make_dir(ExtractDir),
Archive = filename:join(ExtractDir, "abc.zip"),

ok = file:make_dir(ExtractDir),
{ok, _Size} = file:copy(filename:join(DataDir, "abc.zip"), Archive),
FileName = "abc.txt",
Quote = "quotes/rain.txt",
Expand All @@ -726,6 +759,26 @@ unzip_from_binary(Config) when is_list(Config) ->

%% Clean up.
delete_files([DestFilename, DestQuote, Archive, ExtractDir]),

ok = file:make_dir(ExtractDir),
file:set_cwd(ExtractDir),

%% Read a zip file into a binary and extract from the binary with skip_directories
{ok, [FileName,Quote,Wikipedia,EmptyFile]}
= zip:unzip(Bin, [skip_directories]),

%% Verify.
DestFilename = filename:join(ExtractDir, "abc.txt"),
{ok, Data} = file:read_file(filename:join(DataDir, FileName)),
{ok, Data} = file:read_file(DestFilename),

DestQuote = filename:join([ExtractDir, "quotes", "rain.txt"]),
{ok, QuoteData} = file:read_file(filename:join(DataDir, Quote)),
{ok, QuoteData} = file:read_file(DestQuote),

%% Clean up.
delete_files([DestFilename, DestQuote, ExtractDir]),

ok.

%% oac_files() ->
Expand Down

0 comments on commit 193339a

Please sign in to comment.