From 35d0042be1cd83914d3362fafbd1b5d90c4a155a Mon Sep 17 00:00:00 2001 From: bakaq Date: Thu, 21 Sep 2023 21:06:02 -0300 Subject: [PATCH 1/6] Add phrase_from_stream/2 to library(pio) --- src/lib/pio.pl | 170 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 154 insertions(+), 16 deletions(-) diff --git a/src/lib/pio.pl b/src/lib/pio.pl index fdc5bcb1..781f14c8 100644 --- a/src/lib/pio.pl +++ b/src/lib/pio.pl @@ -9,6 +9,7 @@ :- module(pio, [phrase_from_file/2, phrase_from_file/3, + phrase_from_stream/2, phrase_to_file/2, phrase_to_file/3, phrase_to_stream/2 @@ -17,16 +18,30 @@ :- use_module(library(dcgs)). :- use_module(library(error)). :- use_module(library(freeze)). -:- use_module(library(iso_ext), [setup_call_cleanup/3, partial_string/3]). -:- use_module(library(lists), [member/2, maplist/2]). +:- use_module(library(gensym)). +:- use_module(library(iso_ext), [ + bb_get/2, bb_put/2, setup_call_cleanup/3, partial_string/3, partial_string_tail/2 +]). +:- use_module(library(lists), [length/2, member/2, maplist/2]). :- use_module(library(charsio), [get_n_chars/3]). :- meta_predicate(phrase_from_file(2, ?)). :- meta_predicate(phrase_from_file(2, ?, ?)). +:- meta_predicate(phrase_from_stream(2, ?)). :- meta_predicate(phrase_to_file(2, ?)). :- meta_predicate(phrase_to_file(2, ?, ?)). :- meta_predicate(phrase_to_stream(2, ?)). + +%% phrase_from_stream(+GRBody, +Stream) +% +% True if grammar rule body GRBody covers the contents of the stream, +% represented as a list of characters. + +phrase_from_stream(GRBody, Stream) :- + stream_to_lazy_list(Stream, Ls), + phrase(GRBody, Ls). + %% phrase_from_file(+GRBody, +File) % % True if grammar rule body GRBody covers the contents of File, @@ -49,24 +64,147 @@ phrase_from_file(NT, File, Options) :- ; Type = text ), setup_call_cleanup(open(File, read, Stream, [reposition(true)|Options]), - ( stream_to_lazy_list(Stream, Xs), - phrase(NT, Xs) ), + phrase_from_stream(NT, Stream), close(Stream)) - ). - + ). stream_to_lazy_list(Stream, Xs) :- - stream_property(Stream, position(Pos)), - freeze(Xs, reader_step(Stream, Pos, Xs)). + stream_property(Stream, reposition(Rep)), + ( Rep = true -> + stream_to_lazy_list_repositionable(Stream, Xs) + ; stream_to_lazy_list_buffer(Stream, Xs) + ). -reader_step(Stream, Pos, Xs0) :- - set_stream_position(Stream, Pos), - ( at_end_of_stream(Stream) - -> Xs0 = [] - ; get_n_chars(Stream, 4096, Cs), - partial_string(Cs, Xs0, Xs), - stream_to_lazy_list(Stream, Xs) - ). +stream_to_lazy_list_repositionable(Stream, Xs) :- + stream_property(Stream, position(Pos)), + freeze(Xs, reader_step_repositionable(Stream, Pos, Xs)). + +reader_step_repositionable(Stream, Pos, Xs0) :- + set_stream_position(Stream, Pos), + ( at_end_of_stream(Stream) + -> Xs0 = [] + ; get_n_chars(Stream, 4096, Cs), + partial_string(Cs, Xs0, Xs), + stream_to_lazy_list_repositionable(Stream, Xs) + ). + +stream_to_lazy_list_buffer(Stream, Ls) :- + get_stream_buffer_position(Stream, Pos), + freeze(Ls, render_step_buffer(Stream, Pos, Ls)). + +render_step_buffer(Stream, Pos, Ls) :- + set_stream_buffer_position(Stream, Pos), + ( buffer_at_end_of_stream(Stream) -> + Ls = [] + ; buffer_get_n_chars(Stream, 4096, Chars), + partial_string(Chars, Ls, Ls0), + stream_to_lazy_list_buffer(Stream, Ls0) + ). + +buffer_at_end_of_stream(Stream) :- + stream_bufferids(Stream, _, BufferPosId, _), + bb_get(BufferPosId, Pos), + Pos = eof. + +get_stream_buffer_position(Stream, Pos) :- + stream_bufferids(Stream, _, BufferPosId, _), + bb_get(BufferPosId, Pos). + +set_stream_buffer_position(Stream, Pos) :- + stream_bufferids(Stream, _, BufferPosId, _), + bb_put(BufferPosId, Pos). + +buffer_get_n_chars(Stream, N, Chars) :- + stream_bufferids(Stream, BufferId, BufferPosId, BufferLenId), + buffer_prepare_for_n(Stream, BufferId, BufferPosId, BufferLenId, N), + bb_get(BufferId, Buffer), + bb_get(BufferPosId, BufferPos), + ( BufferPos = eof -> + Chars = [] + ; string_get_n_chars(Buffer, BufferPos, N, Chars), + length(Chars, NChars), + ( NChars = 0 -> + BufferPos1 = eof + ; BufferPos1 is BufferPos + NChars + ), + bb_put(BufferPosId, BufferPos1) + ). + +buffer_prepare_for_n(Stream, BufferId, BufferPosId, BufferLenId, N) :- + bb_get(BufferPosId, BufferPos), + bb_get(BufferLenId, BufferLen), + ( BufferLen < BufferPos + N -> + bb_get(BufferId, Buffer), + ( + ( var(Buffer) -> + BufferTail = Buffer + ; partial_string_last_tail(Buffer, BufferTail) + ) -> + ( at_end_of_stream(Stream) -> + BufferTail = [], + bb_put(BufferId, Buffer) + ; get_n_chars(Stream, 4096, Chars), + length(Chars, NChars), + partial_string(Chars, BufferTail, _), + bb_put(BufferId, Buffer), + BufferLen1 is BufferLen + NChars, + bb_put(BufferLenId, BufferLen1), + buffer_prepare_for_n(Stream, BufferId, BufferPosId, BufferLenId, N) + ) + ; true + ) + ; true + ). + +partial_string_last_tail(PartialString, PartialStringTail) :- + partial_string_tail(PartialString, PartialStringTail0), + ( var(PartialStringTail0) -> + PartialStringTail = PartialStringTail0 + ; partial_string_last_tail(PartialStringTail0, PartialStringTail) + ). + +string_get_n_chars([], _, _, []). +string_get_n_chars([S|Ss], BufferPos, N, Chars) :- + ( BufferPos = 0 -> + string_get_n_chars_([S|Ss], N, Chars) + ; BufferPos1 is BufferPos - 1, + string_get_n_chars(Ss, BufferPos1, N, Chars) + ). + +string_get_n_chars_([], _, []). +string_get_n_chars_([S|Ss], N, Chars) :- + ( N = 0 -> + Chars = [] + ; N = 1 -> + % This case is needed to not break the tail of the partial string + Chars = [S] + ; Chars = [S|Cs], + N1 is N - 1, + string_get_n_chars_(Ss, N1, Cs) + ). + +stream_bufferids(Stream, BufferId, BufferPosId, BufferLenId) :- + ( bb_get(streams_buffers, _) -> + true + ; bb_put(streams_buffers, []) + ), + bb_get(streams_buffers, StreamsBuffers), + ( member( + stream_buffer(Stream, BufferId, BufferPosId, BufferLenId), + StreamsBuffers + ) -> + true + ; gensym(buffer, BufferId), + gensym(buffer_pos, BufferPosId), + gensym(buffer_len, BufferLenId), + bb_put( + streams_buffers, + [stream_buffer(Stream, BufferId, BufferPosId, BufferLenId)|StreamsBuffers] + ), + bb_put(BufferId, _), + bb_put(BufferPosId, 0), + bb_put(BufferLenId, 0) + ). %% phrase_to_stream(+GRBody, +Stream) % From b4fab5a80646e3beee6f9a945209c50b78c98ec4 Mon Sep 17 00:00:00 2001 From: bakaq Date: Sat, 23 Sep 2023 01:43:45 -0300 Subject: [PATCH 2/6] Use '$skip_max_list'/4 in string_get_n_chars/4 --- src/lib/pio.pl | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/lib/pio.pl b/src/lib/pio.pl index 781f14c8..8688e1c9 100644 --- a/src/lib/pio.pl +++ b/src/lib/pio.pl @@ -68,6 +68,9 @@ phrase_from_file(NT, File, Options) :- close(Stream)) ). +% How many chars to read from stream and buffer in each step +chars_to_read(4096). + stream_to_lazy_list(Stream, Xs) :- stream_property(Stream, reposition(Rep)), ( Rep = true -> @@ -83,7 +86,8 @@ reader_step_repositionable(Stream, Pos, Xs0) :- set_stream_position(Stream, Pos), ( at_end_of_stream(Stream) -> Xs0 = [] - ; get_n_chars(Stream, 4096, Cs), + ; chars_to_read(CharsToRead), + get_n_chars(Stream, CharsToRead, Cs), partial_string(Cs, Xs0, Xs), stream_to_lazy_list_repositionable(Stream, Xs) ). @@ -96,7 +100,8 @@ render_step_buffer(Stream, Pos, Ls) :- set_stream_buffer_position(Stream, Pos), ( buffer_at_end_of_stream(Stream) -> Ls = [] - ; buffer_get_n_chars(Stream, 4096, Chars), + ; chars_to_read(CharsToRead), + buffer_get_n_chars(Stream, CharsToRead, Chars), partial_string(Chars, Ls, Ls0), stream_to_lazy_list_buffer(Stream, Ls0) ). @@ -143,7 +148,8 @@ buffer_prepare_for_n(Stream, BufferId, BufferPosId, BufferLenId, N) :- ( at_end_of_stream(Stream) -> BufferTail = [], bb_put(BufferId, Buffer) - ; get_n_chars(Stream, 4096, Chars), + ; chars_to_read(CharsToRead), + get_n_chars(Stream, CharsToRead, Chars), length(Chars, NChars), partial_string(Chars, BufferTail, _), bb_put(BufferId, Buffer), @@ -163,13 +169,16 @@ partial_string_last_tail(PartialString, PartialStringTail) :- ; partial_string_last_tail(PartialStringTail0, PartialStringTail) ). -string_get_n_chars([], _, _, []). -string_get_n_chars([S|Ss], BufferPos, N, Chars) :- - ( BufferPos = 0 -> - string_get_n_chars_([S|Ss], N, Chars) - ; BufferPos1 is BufferPos - 1, - string_get_n_chars(Ss, BufferPos1, N, Chars) - ). +string_get_n_chars(String, Pos, N, Chars) :- + chars_to_read(CharsToRead), + ( CharsToRead < Pos -> + % I have absolutely no idea why this is needed (maybe it's a bug?), + % but hey, it works. + '$skip_max_list'(_, Pos, String, String0), + '$skip_max_list'(_, CharsToRead, String0, String1) + ; '$skip_max_list'(_, Pos, String, String1) + ), + string_get_n_chars_(String1, N, Chars). string_get_n_chars_([], _, []). string_get_n_chars_([S|Ss], N, Chars) :- From 2fe79b5fc3343b6245bf4dde1dec748bfe3ef997 Mon Sep 17 00:00:00 2001 From: bakaq Date: Sun, 24 Sep 2023 15:07:39 -0300 Subject: [PATCH 3/6] Fixed bug with '$skip_max_list'/4 --- src/lib/pio.pl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/lib/pio.pl b/src/lib/pio.pl index 8688e1c9..b6fe7786 100644 --- a/src/lib/pio.pl +++ b/src/lib/pio.pl @@ -170,14 +170,7 @@ partial_string_last_tail(PartialString, PartialStringTail) :- ). string_get_n_chars(String, Pos, N, Chars) :- - chars_to_read(CharsToRead), - ( CharsToRead < Pos -> - % I have absolutely no idea why this is needed (maybe it's a bug?), - % but hey, it works. - '$skip_max_list'(_, Pos, String, String0), - '$skip_max_list'(_, CharsToRead, String0, String1) - ; '$skip_max_list'(_, Pos, String, String1) - ), + '$skip_max_list'(_, Pos, String, String1), string_get_n_chars_(String1, N, Chars). string_get_n_chars_([], _, []). From c50291cec8e462061bea65c98f5bce1fafe803d9 Mon Sep 17 00:00:00 2001 From: bakaq Date: Sun, 24 Sep 2023 19:21:03 -0300 Subject: [PATCH 4/6] Better string_get_n_chars_/3 --- src/lib/pio.pl | 46 +++++++++------------------------------------- 1 file changed, 9 insertions(+), 37 deletions(-) diff --git a/src/lib/pio.pl b/src/lib/pio.pl index b6fe7786..24070d73 100644 --- a/src/lib/pio.pl +++ b/src/lib/pio.pl @@ -22,7 +22,7 @@ :- use_module(library(iso_ext), [ bb_get/2, bb_put/2, setup_call_cleanup/3, partial_string/3, partial_string_tail/2 ]). -:- use_module(library(lists), [length/2, member/2, maplist/2]). +:- use_module(library(lists), [append/3, length/2, member/2, maplist/2]). :- use_module(library(charsio), [get_n_chars/3]). :- meta_predicate(phrase_from_file(2, ?)). @@ -71,39 +71,18 @@ phrase_from_file(NT, File, Options) :- % How many chars to read from stream and buffer in each step chars_to_read(4096). -stream_to_lazy_list(Stream, Xs) :- - stream_property(Stream, reposition(Rep)), - ( Rep = true -> - stream_to_lazy_list_repositionable(Stream, Xs) - ; stream_to_lazy_list_buffer(Stream, Xs) - ). - -stream_to_lazy_list_repositionable(Stream, Xs) :- - stream_property(Stream, position(Pos)), - freeze(Xs, reader_step_repositionable(Stream, Pos, Xs)). - -reader_step_repositionable(Stream, Pos, Xs0) :- - set_stream_position(Stream, Pos), - ( at_end_of_stream(Stream) - -> Xs0 = [] - ; chars_to_read(CharsToRead), - get_n_chars(Stream, CharsToRead, Cs), - partial_string(Cs, Xs0, Xs), - stream_to_lazy_list_repositionable(Stream, Xs) - ). - -stream_to_lazy_list_buffer(Stream, Ls) :- +stream_to_lazy_list(Stream, Ls) :- get_stream_buffer_position(Stream, Pos), - freeze(Ls, render_step_buffer(Stream, Pos, Ls)). + freeze(Ls, render_step(Stream, Pos, Ls)). -render_step_buffer(Stream, Pos, Ls) :- +render_step(Stream, Pos, Ls) :- set_stream_buffer_position(Stream, Pos), ( buffer_at_end_of_stream(Stream) -> Ls = [] ; chars_to_read(CharsToRead), buffer_get_n_chars(Stream, CharsToRead, Chars), partial_string(Chars, Ls, Ls0), - stream_to_lazy_list_buffer(Stream, Ls0) + stream_to_lazy_list(Stream, Ls1) ). buffer_at_end_of_stream(Stream) :- @@ -173,17 +152,10 @@ string_get_n_chars(String, Pos, N, Chars) :- '$skip_max_list'(_, Pos, String, String1), string_get_n_chars_(String1, N, Chars). -string_get_n_chars_([], _, []). -string_get_n_chars_([S|Ss], N, Chars) :- - ( N = 0 -> - Chars = [] - ; N = 1 -> - % This case is needed to not break the tail of the partial string - Chars = [S] - ; Chars = [S|Cs], - N1 is N - 1, - string_get_n_chars_(Ss, N1, Cs) - ). +string_get_n_chars_(String, N, Chars) :- + '$skip_max_list'(N1, N, String, _), + length(Chars, N1), + append(Chars, _, String). stream_bufferids(Stream, BufferId, BufferPosId, BufferLenId) :- ( bb_get(streams_buffers, _) -> From 63bb993c02c372a243f07c9815c0ba111009fa3b Mon Sep 17 00:00:00 2001 From: bakaq Date: Sun, 24 Sep 2023 19:29:51 -0300 Subject: [PATCH 5/6] Inline string_get_n_chars_/3 --- src/lib/pio.pl | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/lib/pio.pl b/src/lib/pio.pl index 24070d73..1576ec9a 100644 --- a/src/lib/pio.pl +++ b/src/lib/pio.pl @@ -82,7 +82,7 @@ render_step(Stream, Pos, Ls) :- ; chars_to_read(CharsToRead), buffer_get_n_chars(Stream, CharsToRead, Chars), partial_string(Chars, Ls, Ls0), - stream_to_lazy_list(Stream, Ls1) + stream_to_lazy_list(Stream, Ls0) ). buffer_at_end_of_stream(Stream) :- @@ -150,12 +150,9 @@ partial_string_last_tail(PartialString, PartialStringTail) :- string_get_n_chars(String, Pos, N, Chars) :- '$skip_max_list'(_, Pos, String, String1), - string_get_n_chars_(String1, N, Chars). - -string_get_n_chars_(String, N, Chars) :- - '$skip_max_list'(N1, N, String, _), + '$skip_max_list'(N1, N, String1, _), length(Chars, N1), - append(Chars, _, String). + append(Chars, _, String1). stream_bufferids(Stream, BufferId, BufferPosId, BufferLenId) :- ( bb_get(streams_buffers, _) -> From f644a76281557085b61bc81965955f8e441e9e5c Mon Sep 17 00:00:00 2001 From: bakaq Date: Mon, 25 Sep 2023 02:18:05 -0300 Subject: [PATCH 6/6] Remove reposition option from phrase_from_file/2 --- src/lib/pio.pl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib/pio.pl b/src/lib/pio.pl index 1576ec9a..f13c9b85 100644 --- a/src/lib/pio.pl +++ b/src/lib/pio.pl @@ -63,9 +63,11 @@ phrase_from_file(NT, File, Options) :- member(Type, [text,binary]) ; Type = text ), - setup_call_cleanup(open(File, read, Stream, [reposition(true)|Options]), - phrase_from_stream(NT, Stream), - close(Stream)) + setup_call_cleanup( + open(File, read, Stream, Options), + phrase_from_stream(NT, Stream), + close(Stream) + ) ). % How many chars to read from stream and buffer in each step