From 63fb669803b3af87f5a2be970f97aac8a2581998 Mon Sep 17 00:00:00 2001 From: Fritz Zaucker Date: Mon, 4 Sep 2023 15:33:43 +0200 Subject: [PATCH 01/21] Use auto-deleting temp file for excel export --- lib/Agrammon/OutputFormatter/ExcelFast.pm6 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/Agrammon/OutputFormatter/ExcelFast.pm6 b/lib/Agrammon/OutputFormatter/ExcelFast.pm6 index 8bfb5569..ccfa01d7 100644 --- a/lib/Agrammon/OutputFormatter/ExcelFast.pm6 +++ b/lib/Agrammon/OutputFormatter/ExcelFast.pm6 @@ -8,6 +8,7 @@ use Agrammon::Web::SessionUser; #use Spreadsheet::XLSX; #use Spreadsheet::XLSX::Styles; +use Temp::Path; use Excel::Writer::XLSX:from; sub input-output-as-excel( @@ -19,9 +20,8 @@ sub input-output-as-excel( Bool $include-filters, Bool $all-filters ) is export { - my $temp-filename = "$dataset-name.xlsx"; - - my $workbook = Excel::Writer::XLSX.new($temp-filename); + my $temp-file = make-temp-path :suffix<.xlsx>; + my $workbook = Excel::Writer::XLSX.new($temp-file.absolute); # prepare sheets my $output-sheet = $workbook.add_worksheet('Ergebnisse'); @@ -162,5 +162,5 @@ sub input-output-as-excel( $row-formatted++; } $workbook.close(); - return $temp-filename.IO.slurp: :bin; + return $temp-file.slurp: :bin; } From 1534e4ce1d6d9f802bda839151b00da34cadfe3f Mon Sep 17 00:00:00 2001 From: Fritz Zaucker Date: Mon, 4 Sep 2023 15:34:35 +0200 Subject: [PATCH 02/21] Remove double extension from Excel export --- lib/Agrammon/UI/CommandLine.rakumod | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/Agrammon/UI/CommandLine.rakumod b/lib/Agrammon/UI/CommandLine.rakumod index 7c6fb96d..4bbeccc7 100644 --- a/lib/Agrammon/UI/CommandLine.rakumod +++ b/lib/Agrammon/UI/CommandLine.rakumod @@ -59,7 +59,7 @@ multi sub MAIN('web', Str $model-filename, ExistingFile :$cfg-file, Str :$techni multi sub MAIN('run', Str $filename, ExistingFileOrStdin $input, ExistingFile :$cfg-file, Str :$technical-file, SupportedLanguage :$language = 'de', Str :$print-only, Int :$report-selected, Str :$variants = 'Base', Bool :$include-filters, Bool :$include-all-filters=False, Int :$batch=1, Int :$degree=4, Int :$max-runs, - OutputFormat :$format = 'text', Str :$export-filename='test.xlsx' + OutputFormat :$format = 'text', Str :$export-filename='test' ) is export { my @print-set = $print-only.split(',') if $print-only; my $data = run $filename, $input.IO, $technical-file, $variants, $format, $language, @print-set, @@ -98,8 +98,8 @@ multi sub MAIN('run', Str $filename, ExistingFileOrStdin $input, ExistingFile :$ } for %sim-results.keys.sort -> $dataset { if $format eq 'excel' { - spurt "$export-filename.xlsx", %sim-results{$dataset}; - return; + $export-filename //= $dataset; + spurt "$export-filename-$dataset.xlsx", %sim-results{$dataset}; } else { @output.push("# Dataset $dataset"); From ef3472aa9afa3129de7e6ee22a07de2763c74d3f Mon Sep 17 00:00:00 2001 From: Fritz Zaucker Date: Mon, 4 Sep 2023 15:35:26 +0200 Subject: [PATCH 03/21] Use ExcelFast properly in GUI --- lib/Agrammon/Web/Routes.rakumod | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/Agrammon/Web/Routes.rakumod b/lib/Agrammon/Web/Routes.rakumod index d91b589c..0b2887fa 100644 --- a/lib/Agrammon/Web/Routes.rakumod +++ b/lib/Agrammon/Web/Routes.rakumod @@ -357,7 +357,9 @@ sub frontend-api-routes (Str $schema, $ws) { request-body -> %params { # prevent header injection my $filename = cleanup-filename "%params.xlsx"; - my $excel = $ws.get-excel-export($user, %params).to-blob; + # use with Excel.pm + # my $excel = $ws.get-excel-export($user, %params).to-blob; + my $excel = $ws.get-excel-export($user, %params); header 'Content-disposition', qq{attachment; filename="$filename"}; content 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', $excel; CATCH { From 035f9ad92a6427436f9351411d7dffac1f467ca3 Mon Sep 17 00:00:00 2001 From: Fritz Zaucker Date: Mon, 4 Sep 2023 17:21:20 +0200 Subject: [PATCH 04/21] Update dependencies --- dev/META6.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/META6.json b/dev/META6.json index 3742bdae..18977cc6 100644 --- a/dev/META6.json +++ b/dev/META6.json @@ -21,6 +21,7 @@ "DB::Pg", "Digest::SHA1::Native", "Email::MIME", + "Inline::Perl5", "IO::Path::ChildSecure", "IO::String", "LibXML:ver<0.7.9+>", @@ -29,6 +30,7 @@ "OO::Monitors", "Shell::Command", "Spreadsheet::XLSX:ver<0.2.4+>", + "Temp::Path", "Text::CSV", "YAMLish", "Cro::APIToken", From ba9e21cb8c224cc2f2b26c41afdcae92652bf088 Mon Sep 17 00:00:00 2001 From: Fritz Zaucker Date: Tue, 5 Sep 2023 10:52:52 +0200 Subject: [PATCH 05/21] Update configure/make setup for Perl5 modules --- .gitignore | 8 +++++++ .tool-versions | 1 + Makefile.am | 34 +++++++------------------- VERSION | 2 +- configure.ac | 1 + cpanfile | 1 + thirdparty/Makefile.am | 54 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 75 insertions(+), 26 deletions(-) create mode 100644 .tool-versions create mode 100644 cpanfile create mode 100644 thirdparty/Makefile.am diff --git a/.gitignore b/.gitignore index 686f558c..844400cd 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,11 @@ etc/*.yaml etc/*.secret public/ conftools/ +thirdparty/build.log +thirdparty/cache +thirdparty/carton/ +thirdparty/latest-build +thirdparty/touch +thirdparty/lib/ +thirdparty/work/ +thirdparty/bin/ diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 00000000..92d82d03 --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +perl system diff --git a/Makefile.am b/Makefile.am index 32c4ba57..60e117d3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,33 +1,17 @@ -# Copyright (C) 2020 Fritz Zaucker +# Copyright (C) 2023 Fritz Zaucker -AUTOMAKE_OPTIONS = foreign +AUTOMAKE_OPTIONS = foreign -SUBDIRS = etc frontend -PUB := $(shell find -L public -type d \( -name ".??*" -o -name transpiled \) -prune -o -not -name "*db.json" -a -not -name "*.map" -a -not -name "*~" -a -not -name transpiled -a -not -name "*.tmp" -a -type f -print ) -SHARE := $(shell test -d share && find -L share -type d -name ".??*" -prune -o -not -name ".*" -a -not -name "*~" -a -not -name "*.tmp" -a -not -name '*.sql' -a -not -wholename '*/Docu/*' -a -not -wholename '*/Docu' -a -not -wholename '*/_Attic' -a -not -wholename '*/_Attic/*' -a -type f -print) - -PERLTESTS := $(shell find -L t -name "*.t") -PM := $(shell find -L lib -name "*.pm") - -#EXTRA_DIST = META6.json COPYRIGHT LICENSE CHANGES AUTHORS bootstrap $(PUB) $(POD) $(TEMPL) $(SHARE) - -EXTRA_DIST = META6.json COPYRIGHT bin lib/Agrammon bootstrap $(PUB) $(SHARE) # $(PERLTESTS) +SUBDIRS = thirdparty etc frontend -YEAR := $(shell date +%Y) -DATE := $(shell date +%Y-%m-%d) - -datadir = $(prefix) -nobase_data_DATA = $(PUB) $(TEMPL) $(SHARE) - -README.md COPYRIGHT: - $(PERL) -i -p -e 's/(#VERSION#|\d+\.\d+\.\d+[^.\s]*)/$(PACKAGE_VERSION)/g;s/(#YEAR#|20[1-9]\d)/$(YEAR)/g;s/(#DATE#|20[1-9]\d-[01]\d-[0-3]\d)/$(DATE)/g;' $@ +PUB := $(shell find -L public -type d \( -name ".??*" -o -name transpiled \) -prune -o -not -name "*db.json" -a -not -name "*.map" -a -not -name "*~" -a -not -name transpiled -a -not -name "*.tmp" -a -type f -print ) +TEMPL := $(shell test -d templates && find -L templates -type f -name "*.ep") +SHARE := $(shell test -d share && find -L share -type d -name ".??*" -prune -o -not -name ".*" -a -not -name "*~" -a -not -name "*.tmp" -a -type f -print) -LANGUAGES := $(shell $(PERL) -e 'use JSON::PP qw(decode_json); print join(" ", map {"share/".$$_.".po"} @{decode_json(join("",<>))->{locales}})' frontend/compile.json) +PERLTESTS := $(shell find t -name "*.t") +PM := $(shell find lib -name "*.pm") -test: - prove -j8 -e 'raku -Ilib' t +EXTRA_DIST = VERSION cpanfile COPYRIGHT LICENSE CHANGES AUTHORS bootstrap $(PUB) $(wildcard t/*.t) $(POD) $(TEMPL) $(PERLTESTS) $(SHARE) Dockerfile -unit-test: - AGRAMMON_UNIT_TEST=1 prove -j8 -e 'raku -Ilib' t diff --git a/VERSION b/VERSION index 798e3899..19b860c1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.3.0 +6.4.0 diff --git a/configure.ac b/configure.ac index 36df524a..b64433df 100644 --- a/configure.ac +++ b/configure.ac @@ -154,6 +154,7 @@ AC_CONFIG_FILES([ Makefile etc/Makefile frontend/Makefile + thirdparty/Makefile ]) AC_SUBST(VERSION) diff --git a/cpanfile b/cpanfile new file mode 100644 index 00000000..ce89cddd --- /dev/null +++ b/cpanfile @@ -0,0 +1 @@ +requires 'Excel::Writer::XLSX'; diff --git a/thirdparty/Makefile.am b/thirdparty/Makefile.am new file mode 100644 index 00000000..6efed1e6 --- /dev/null +++ b/thirdparty/Makefile.am @@ -0,0 +1,54 @@ +# Copyright (C) 2023 Fritz Zaucker + +AUTOMAKE_OPTIONS = foreign + +THIRDPARTY_DIR := $(shell pwd) + +CPANSNAPV := cpanfile-$(shell $(PERL) -MConfig -e 'my $$v = $$Config{version}; $$v =~ s/\.\d+$$//; print $$v').snapshot + + +#EXTRA_DIST = $(THIRDPARTY_DIST) $(wildcard bin/cpanm) +EXTRA_DIST = bin/cpanm $(wildcard cpanfile*snapshot) + +all-local: touch + +touch: bin/cpanm $(CPANSNAPV) + $(AM_V_at)echo "** Installing Dependencies using $(CPANSNAPV)" + cp $(CPANSNAPV) ../cpanfile.snapshot + test -x carton/bin/carton || PERL_CPANM_OPT= PERL_CPANM_HOME=$(THIRDPARTY_DIR) $(PERL) bin/cpanm -q --notest --local-lib-contained $(THIRDPARTY_DIR)/carton Carton Date::Parse +# if ever DBD::ODBC is compiled, make sure we get the utf8 version + PERL_CPANM_OPT= PERL_CPANM_HOME=$(THIRDPARTY_DIR) DBD_ODBC_UNICODE=1 PERL5LIB=$(THIRDPARTY_DIR)/carton/lib/perl5 PERL_CARTON_PATH=$(THIRDPARTY_DIR) $(PERL) $(THIRDPARTY_DIR)/carton/bin/carton install + $(AM_V_at)rm -f ../cpanfile.snapshot + $(AM_V_at)touch touch + +bin/cpanm: + $(AM_V_at)mkdir -p bin + $(URL_CAT) https://cpanmin.us > bin/cpanm + $(AM_V_at)chmod 755 bin/cpanm + +$(CPANSNAPV): ../cpanfile + $(AM_V_at)echo "** Installing Dependencies using Carton install" + test -f $(CPANSNAPV) && cp $(CPANSNAPV) ../cpanfile.snapshot || true + test -x carton/bin/carton || PERL_CPANM_OPT= PERL_CPANM_HOME=$(THIRDPARTY_DIR) $(PERL) bin/cpanm -q --notest --local-lib-contained $(THIRDPARTY_DIR)/carton Carton Date::Parse +# if ever DBD::ODBC is compiled, make sure we get the utf8 version + PERL_CPANM_OPT= PERL_CPANM_HOME=$(THIRDPARTY_DIR) DBD_ODBC_UNICODE=1 PERL5LIB=$(THIRDPARTY_DIR)/carton/lib/perl5 PERL_CARTON_PATH=$(THIRDPARTY_DIR) $(PERL) $(THIRDPARTY_DIR)/carton/bin/carton install + mv ../cpanfile.snapshot $(CPANSNAPV) + $(AM_V_at)touch touch + +update: $(CPANSNAPV) + $(AM_V_at)echo "** Updating Dependencies using Carton update" + $(AM_V_at)cp $(CPANSNAPV) ../cpanfile.snapshot + $(AM_V_at)PERL_CPANM_OPT= PERL_CPANM_HOME=$(THIRDPARTY_DIR) PERL5LIB=$(THIRDPARTY_DIR)/carton/lib/perl5 PERL_CARTON_PATH=$(THIRDPARTY_DIR) $(PERL) $(THIRDPARTY_DIR)/carton/bin/carton update + $(AM_V_at)mv ../cpanfile.snapshot $(CPANSNAPV) + +clean-local: + ls -1 | grep -v Makefile | grep -v cpanfile |grep -v bin | xargs rm -rf + +distclean-local: + ls -1 | grep -v Makefile | grep -v cpanfile | xargs rm -rf + +install-exec-hook: + cp -fr lib/perl5/* $(DESTDIR)$(libdir) + cp -f bin/cover $(DESTDIR)$(bindir) + $(AM_V_at)echo "** Patching cover command" + $(PERL) -i -p -e 's{use warnings;}{use warnings;\nuse lib qw($(libdir));}' $(DESTDIR)$(bindir)/cover From abc9903f24910e680c3d3924018dedc6f3bec12a Mon Sep 17 00:00:00 2001 From: Fritz Zaucker Date: Fri, 22 Mar 2024 17:46:25 +0100 Subject: [PATCH 06/21] Add ExcelFast module to repo --- .../OutputFormatter/ExcelFast.rakumod | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 lib/Agrammon/OutputFormatter/ExcelFast.rakumod diff --git a/lib/Agrammon/OutputFormatter/ExcelFast.rakumod b/lib/Agrammon/OutputFormatter/ExcelFast.rakumod new file mode 100644 index 00000000..1d8d14bf --- /dev/null +++ b/lib/Agrammon/OutputFormatter/ExcelFast.rakumod @@ -0,0 +1,187 @@ +use v6; +use Agrammon::Config; +use Agrammon::Model; +use Agrammon::Outputs; +use Agrammon::OutputFormatter::CollectData; +use Agrammon::Timestamp; +use Agrammon::Web::SessionUser; +#use Spreadsheet::XLSX; +#use Spreadsheet::XLSX::Styles; + +use Temp::Path; +use Excel::Writer::XLSX:from; + +sub input-output-as-excel( + Agrammon::Config $cfg, + $user, + Str $dataset-name, Agrammon::Model $model, + Agrammon::Outputs $outputs, Agrammon::Inputs $inputs, $reports, + Str $language, Int $report-selected, + Bool $include-filters, Bool $all-filters +) is export { + + my $temp-file = make-temp-path :suffix<.xlsx>; + my $workbook = Excel::Writer::XLSX.new($temp-file.absolute); + + # prepare sheets + my $output-sheet = $workbook.add_worksheet('Ergebnisse'); + my $output-sheet-formatted = $workbook.add_worksheet('Ergebnisse formatiert'); + my $input-sheet = $workbook.add_worksheet('Eingaben'); + my $input-sheet-formatted = $workbook.add_worksheet('Eingaben formatiert'); + my $input-sheet-raw = $workbook.add_worksheet('Eingaben für REST'); + + my $timestamp = timestamp; + my $model-version = $cfg.gui-title{$language} ~ " - " ~ $cfg.gui-variant; + + # set column width + for ($output-sheet, $output-sheet-formatted) -> $sheet { + $sheet.set_column(0, 0, 20); + $sheet.set_column(1, 1, 32); + $sheet.set_column(2, 2, 20); + $sheet.set_column(3, 3, 10); + + } + + $input-sheet.set_column(0, 0, 30); + $input-sheet.set_column(1, 1, 20); + $input-sheet.set_column(2, 3, 50); + $input-sheet.set_column(4, 4, 10); + + $input-sheet-raw.set_column(0, 0, 60); + $input-sheet-raw.set_column(1, 1, 50); + $input-sheet-raw.set_column(2, 2, 50); + + $input-sheet-formatted.set_column(0, 0, 10); + $input-sheet-formatted.set_column(1, 2, 50); + $input-sheet-formatted.set_column(3, 3, 10); + + my $bold-format = $workbook.add_format(); + $bold-format.set_bold(); + + my $number-format = $workbook.add_format(); + $number-format.set_num_format( '0.000' ); + + my $number-format-short = $workbook.add_format(); + $number-format-short.set_num_format( '0.0' ); + + my $number-format-right = $workbook.add_format(); + $number-format-right.set_num_format( '0.000' ); + $number-format-right.set_align( 'right' ); + + my $number-format-right-short = $workbook.add_format(); + $number-format-right-short.set_num_format( '0.0' ); + $number-format-right-short.set_align( 'right' ); + + for ($output-sheet-formatted, $input-sheet-formatted) -> $sheet { + $sheet.write(0, 0, $dataset-name, $bold-format); + $sheet.write(1, 0, $user.username); + $sheet.write(2, 0, $model-version); + $sheet.write(3, 0, $timestamp); + } + + for ($output-sheet, $input-sheet) -> $sheet { + $sheet.write(0, 0, $dataset-name); + $sheet.write(1, 0, $user.username); + $sheet.write(2, 0, $model-version); + $sheet.write(3, 0, $timestamp); + } + $input-sheet-raw.write(0,0, "# $dataset-name, {$user.username}, $model-version, $timestamp"); + + # prepared data + my %data = collect-data( + $model, + $outputs, $inputs, $reports, + $language, $report-selected, + $include-filters, $all-filters + ); + + my @records; + # TODO: fix sorting + my $col = 0; + my $row = 5; + my $row-formatted = $row; + my $row-raw = 1; + my $last-print = ''; +# for @records.sort(+*.) -> %rec { + my $last-instance = ''; + my $last-module = ''; + @records := %data; + note "inputs: " ~ @records.elems if %*ENV; + for @records -> %rec { + + # unformatted data + $input-sheet.write($row, $col+0, %rec); + $input-sheet.write($row, $col+1, %rec); + $input-sheet.write($row, $col+2, %rec); + $input-sheet.write($row, $col+3, (%rec // '???'), $number-format-right); + $input-sheet.write($row, $col+4, %rec); + $row++; + + # formatted data + my $instance = %rec; + my $module = %rec; + if $module ne $last-module { + $input-sheet-formatted.write($row-formatted, $col+0, $module, $bold-format); + $row-formatted++; + $last-module = $module; + } + if $instance and $instance ne $last-instance { + $input-sheet-formatted.write($row-formatted, $col+1, $instance, $bold-format); + $row-formatted++; + $last-instance = $instance; + } + $input-sheet-formatted.write($row-formatted, $col+1, %rec); + $input-sheet-formatted.write($row-formatted, $col+2, (%rec // '???'), $number-format-right-short); + $input-sheet-formatted.write($row-formatted, $col+3, %rec); + $row-formatted++; + + # raw data + my $module-instance = %rec; + my $gui = %rec; + if $instance { +# note "instance=$instance, gui=$gui, module-instance=$module-instance"; + my $match = $module-instance.match(/$gui/); + $module-instance = $match.replace-with("$gui\[$instance\]"); + } + $input-sheet-raw.write($row-raw, $col+0, $module-instance); + $input-sheet-raw.write($row-raw, $col+1, %rec); + $input-sheet-raw.write($row-raw, $col+2, (%rec // '???'), $number-format-right); + $row-raw++; + + } + + # add outputs + my %print-labels = %data; + + @records := %data; + note "outputs: " ~ @records.elems if %*ENV; + $row = 5; + $row-formatted = $row; + $col = 0; + $last-print = ''; + + for @records.sort(+*.) -> %rec { + my $print = %rec; # can be undefined or empty + $output-sheet.write($row, $col+0, %print-labels{$print}{$language} // '') if $print; + $output-sheet.write($row, $col+1, %rec