From cf2dfaea1795e5523f096deac46b08fac71303e2 Mon Sep 17 00:00:00 2001 From: Jason Crome Date: Thu, 25 Apr 2024 14:31:02 -0400 Subject: [PATCH] Initial version Docs still need additional work prior to release. --- .gitignore | 23 + Changes | 4 + META.yml | 31 + Makefile.PL | 27 + README | 254 +++++ bin/rt-extension-import-csv.in | 272 +++++ inc/Module/Install.pm | 451 ++++++++ inc/Module/Install/Base.pm | 83 ++ inc/Module/Install/Can.pm | 163 +++ inc/Module/Install/Fetch.pm | 93 ++ inc/Module/Install/Include.pm | 34 + inc/Module/Install/Makefile.pm | 418 ++++++++ inc/Module/Install/Metadata.pm | 722 +++++++++++++ inc/Module/Install/RTx.pm | 316 ++++++ inc/Module/Install/RTx/Runtime.pm | 80 ++ inc/Module/Install/ReadmeFromPod.pm | 184 ++++ inc/Module/Install/Substitute.pm | 131 +++ inc/Module/Install/Win32.pm | 64 ++ inc/Module/Install/WriteAll.pm | 63 ++ inc/YAML/Tiny.pm | 872 +++++++++++++++ lib/RT/Extension/Import/CSV.pm | 1552 +++++++++++++++++++++++++++ 21 files changed, 5837 insertions(+) create mode 100644 .gitignore create mode 100644 Changes create mode 100644 META.yml create mode 100644 Makefile.PL create mode 100644 README create mode 100755 bin/rt-extension-import-csv.in create mode 100644 inc/Module/Install.pm create mode 100644 inc/Module/Install/Base.pm create mode 100644 inc/Module/Install/Can.pm create mode 100644 inc/Module/Install/Fetch.pm create mode 100644 inc/Module/Install/Include.pm create mode 100644 inc/Module/Install/Makefile.pm create mode 100644 inc/Module/Install/Metadata.pm create mode 100644 inc/Module/Install/RTx.pm create mode 100644 inc/Module/Install/RTx/Runtime.pm create mode 100644 inc/Module/Install/ReadmeFromPod.pm create mode 100644 inc/Module/Install/Substitute.pm create mode 100644 inc/Module/Install/Win32.pm create mode 100644 inc/Module/Install/WriteAll.pm create mode 100644 inc/YAML/Tiny.pm create mode 100644 lib/RT/Extension/Import/CSV.pm diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..85cdc54 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +blib* +Makefile +Makefile.old +pm_to_blib* +*.tar.gz +.lwpcookies +cover_db +pod2htm*.tmp +/RT-Extension-Import-CSV* +*.bak +*.swp +/MYMETA.* +/t/tmp +/xt/tmp +bin/rt-extension-import-csv +perllocal.pod +install.json +MYMETA.json +.packlist +local/lib/perl5/x86_64-linux/Text/CSV_XS.pm +local/lib/perl5/x86_64-linux/auto/Text/CSV_XS/CSV_XS.so +bin/rt-extension-import-csv + diff --git a/Changes b/Changes new file mode 100644 index 0000000..f3f92ea --- /dev/null +++ b/Changes @@ -0,0 +1,4 @@ +Revision history for RT-Extension-Import-CSV + +0.01 2024-04-25 +- Initial version diff --git a/META.yml b/META.yml new file mode 100644 index 0000000..8dd43f5 --- /dev/null +++ b/META.yml @@ -0,0 +1,31 @@ +--- +abstract: 'RT-Extension-Import-CSV Extension' +author: + - 'Best Practical Solutions, LLC ' +build_requires: + ExtUtils::MakeMaker: 6.59 +configure_requires: + ExtUtils::MakeMaker: 6.59 +distribution_type: module +dynamic_config: 1 +generated_by: 'Module::Install version 1.19' +license: gpl_2 +meta-spec: + url: http://module-build.sourceforge.net/META-spec-v1.4.html + version: 1.4 +name: RT-Extension-Import-CSV +no_index: + directory: + - etc + - inc +requires: + Test::MockTime: 0 + Text::CSV_XS: 0 + perl: 5.10.1 +resources: + license: http://opensource.org/licenses/gpl-license.php + repository: https://github.com/bestpractical/rt-extension-import-csv +version: '0.01' +x_module_install_rtx_version: '0.42' +x_requires_rt: 5.0.0 +x_rt_too_new: 5.2.0 diff --git a/Makefile.PL b/Makefile.PL new file mode 100644 index 0000000..35cb1d9 --- /dev/null +++ b/Makefile.PL @@ -0,0 +1,27 @@ +use lib '.'; +use inc::Module::Install; + +RTx 'RT-Extension-Import-CSV'; +license 'gpl_2'; +repository 'https://github.com/bestpractical/rt-extension-import-csv'; + +requires_rt '5.0.0'; +rt_too_new '5.2.0'; +requires 'Text::CSV_XS'; +requires 'Test::MockTime'; + +use Config; +my $perl_path = $Config{perlpath}; +$perl_path .= $Config{_exe} + if $^O ne 'VMS' and $perl_path !~ m/$Config{_exe}$/i; + +substitute( { + RT_LIB_PATH => "$RT::LocalPath/lib " . File::Basename::dirname( $INC{'RT.pm'} ), + PERL => $perl_path, + }, + { sufix => ".in" }, + 'bin/rt-extension-import-csv', +); + +sign; +WriteAll; diff --git a/README b/README new file mode 100644 index 0000000..176c69a --- /dev/null +++ b/README @@ -0,0 +1,254 @@ +NAME + RT-Extension-Import-CSV + +DESCRIPTION + Import data into RT from CSVs. + +REQUIREMENTS + Perl module Text::CSV_XS + +RT VERSION + Works with RT 5. + +INSTALLATION + perl Makefile.PL + make + make install + May need root permissions + + Edit your /opt/rt5/etc/RT_SiteConfig.pm + Add this line: + + Plugin('RT::Extension::Import::CSV'); + + Clear your mason cache + rm -rf /opt/rt5/var/mason_data/obj + + Restart your webserver + +CONFIGURATION + The following configuration would be used to import a three-column CSV + of tickets, where the custom field Original Ticket ID must be unique. + That option can accept multiple values and the combination of values + must find no existing tickets for insert, or a single ticket for update. + If multiple tickets match, the CSV row is skipped. + + Set( @TicketsImportUniqueCFs, ('Original Ticket ID') ); + Set( %TicketsImportFieldMapping, + 'Created' => 'Ticket-Create-Date', + 'CF.Original Ticket ID' => 'TicketID', + 'Subject' => 'name', + ); + + Excluding Existing Tickets By Status + Some tickets will be opened, issues will be fixed, and the ticket will + be marked as closed. Later, the same asset (e.g., a server) may have a + new ticket opened for a newly found issue. In these cases, a new ticket + should be created and the previous ticket should not be re-opened. To + instruct the importer to exclude tickets in some statuses, set the + following option: + + Set( @ExcludeStatusesOnSearch, ('reported_fixed')); + + Constant values + If you want to set an RT column or custom field to a static value for + all imported tickets, precede the "CSV field name" (right hand side of + the mapping) with a slash, like so: + + Set( %TicketsImportFieldMapping, + 'Queue' => \'General', + 'Created' => 'Ticket-Create-Date', + 'CF.Original TicketID' => 'TicketID', + 'Subject' => 'name', + ); + + Every imported ticket will now be added to the 'General' queue. This + feature is particularly useful for setting the queue, but may also be + useful when importing tickets from CSV sources you don't control (and + don't want to modify each time). + + Computed values + You may also compute values during import, by passing a subroutine + reference as the value in the %TicketsImportFieldMapping. This + subroutine will be called with a hash reference of the parsed CSV row. + In the following example, the subroutine assigned to the 'Status' field + takes the value in the 'status' CSV column and replaces underscores with + spaces. + + Set( %TicketsImportFieldMapping, + 'Queue' => \'General', + 'Created' => 'Ticket-Create-Date', + 'CF.Original TicketID' => 'TicketID', + 'Subject' => 'name', + 'Status' => sub { $_[0]->{status} =~ s/_/ /gr; }, + ); + + Using computed columns may cause false-positive "unused column" + warnings; these can be ignored. + + Mandatory fields + To mark some ticket fields mandatory: + + Set( @TicketMandatoryFields, 'CF.Severity' ); + + Then rows without "CF.Seveirty" values will be skipped. + + Extra Options for Text::CSV_XS + The CSV importer is configured to read the CSV import format determined + when initially testing. However, the Text::CSV_XS module is configurable + and can handle different CSV variations. You can pass through custom + options using the configuration below. Available options are described + in the documentation for Text::CSV_XS. + + Set( %CSVOptions, ( + binary => 1, + sep_char => ';', + quote_char => '`', + escape_char => '`', + ) ); + + Operations before Create or Update + The importer provides a callback to run operations before a ticket has + been created or updated from CSV content. To run some code before an + update, add the following to your CSV configuration file: + + Set($PreTicketChangeCallback, + sub { + my %args = ( + TicketObj => undef, + Row => undef, + Type => undef, + CurrentUser => undef, + @_, + ); + return 1; # to continue processing current row + } + ); + + As shown, you receive the ticket object(only for "Update" type), the + current CSV row, and the type of update, "Create" or "Update". + CurrentUser is also passed as it may be needed to call other methods. + You can run any code in the callback. + + The Row argument is a reference to a hash with the values from the CSV + file. The keys are the columns from the file and match the CSV import + configuration. The values are for the row currently being processed. + + Since the Row argument is a reference, you can modify the value before + it is processed. For example, to lower case incoming status values, you + could do this: + + if ( exists $args{'Row'}->{status} ) { + $args{'Row'}->{status} = lc($args{'Row'}->{status}); + } + + If you return a false value, the change for that row is skipped, e.g. + + return ( 0, "Obsolete data" ); + + Return a true value to process that row normally. + + return 1; + + Operations after Create or Update + The importer provides a callback to run operations after a ticket has + been created or updated from CSV content. To run some code after an + update, add the following to your CSV configuration file: + + Set($PostTicketChangeCallback, + sub { + my %args = ( + TicketObj => undef, + Row => undef, + Type => undef, + CurrentUser => undef, + @_, + ); + } + ); + + As shown, you receive the ticket object, the current CSV row, and the + type of update, "Create" or "Update". CurrentUser is also passed as it + may be needed to call other methods. You can run any code in the + callback. It expects no return value. + + Special Columns + Comment or Correspond + To add a comment or correspond (reply) to a ticket, you can map a + CSV column to "Comment" or "Correspond". When creating a ticket + (--insert) you can use either one and the content will be added to + the Create transaction. + + TicketsImportTicketIdField + If the CSV data contains the ids of existing RT tickets, you can set + this option to the name of the column containing the RT ticket id. The + importer will then search for that ticket id and update the ticket data + with CSV values. + + Set($TicketsImportTicketIdField, 'RT ticket id'); + + Only one of TicketsImportTicketIdField or @TicketsImportUniqueCFs can be + used for a given CSV file. Also, this option is only valid for --update + or --insert-update modes. You cannot specify the ticket id to be created + in --insert mode. + + TicketTolerantRoles + By default, if a user can't be loaded via LDAP for a role, like Owner, + the importer will log it and skip creating the ticket. For roles that do + not require a successfully loaded user, set this option with the role + name. The importer will then log the failed attempt to find the user, + but still create the ticket. + + Set(@TicketTolerantRoles, 'CR.Subscribers Peers'); + + TransactionsImportFieldMapping + Set the column mappings for importing transactions from a CSV file. A + 'TicketID' mapping is required for RT to add the transaction to an + existing ticket. The 'TicketID' value is mapped to the custom field + 'Original Ticket ID'. + + Attachments can be included by providing the file system path for an + attachment. + + Set( %TransactionsImportFieldMapping, + 'Attachment' => 'Attachment', + 'TicketID' => 'SomeID', + 'Date' => 'Created', + 'Type' => 'Type', + 'Content' => 'Content', + 'AttachmentType' => 'FileType' + ); + +EXECUTION + To import tickets from a CSV file, run the following command: + + local/plugins/RT-Extension-Import-CSV/bin/rt-extension-import-csv \ + --config /full/path/to/the/config/file/tickets-config.pm \ + --type ticket \ + /full/path/to/the/csv/file/tickets.csv + + Note: full path to the config file and CSV files are required + + To import transactions from a CSV file, run the following command: + + local/plugins/RT-Extension-Import-CSV/bin/rt-extension-import-csv \ + --config /full/path/to/the/config/file/transactions-config.pm \ + --type transaction \ + /full/path/to/the/csv/file/transactions.csv + + Note: full path to the config file and CSV files are required + +AUTHOR + Best Practical Solutions, LLC + + All bugs should be reported via email to + bug-RT-Extension-Import-CSV@rt.cpan.org + or via the web at + http://rt.cpan.org/Public/Dist/Display.html?Name=RT-Extension-Import-CSV +LICENSE AND COPYRIGHT + This software is Copyright (c) 2021 by Best Practical LLC + + This is free software, licensed under: + + The GNU General Public License, Version 2, June 1991 + diff --git a/bin/rt-extension-import-csv.in b/bin/rt-extension-import-csv.in new file mode 100755 index 0000000..c6f5579 --- /dev/null +++ b/bin/rt-extension-import-csv.in @@ -0,0 +1,272 @@ +#!/usr/bin/env perl +### before: #!@PERL@ + +use strict; +use warnings; + +### after: use lib qw(@RT_LIB_PATH@); +use lib qw(/opt/rt5/local/lib /opt/rt5/lib); + +use Getopt::Long; +my %opt; +GetOptions( \%opt, 'help|h', 'type|t=s', 'update|u', 'insert|i', 'insert-update', 'force|f', 'debug|d', 'mdy', 'dmy', 'config|c=s', 'run-scrips', 'article-class=s' ); +my $file = shift @ARGV; +my $additional_args = shift @ARGV; + +require Pod::Usage; + +if ( $additional_args ){ + Pod::Usage::pod2usage( "Multiple file names provided. Re-run with only one filename." ); + exit; +} + +if ( $opt{help} || !$file ) { + Pod::Usage::pod2usage( { verbose => 2 } ); + exit; +} + +unless ( $opt{config} ) { + Pod::Usage::pod2usage( "Option config is required" ); + exit; +} + +unless ( ( $opt{type} // '' ) =~ /^(?:user|ticket|transaction|article)$/ ) { + Pod::Usage::pod2usage( "Invalid type provided, should be one of user/ticket/transaction/article" ); + exit; +} + +if ($opt{mdy} and $opt{dmy}) { + Pod::Usage::pod2usage("Only one of --mdy or --dmy can be provided"); + exit; +} + +if ($opt{type} eq 'article' and !$opt{'article-class'}) { + Pod::Usage::pod2usage("--article-class is required for article type"); + exit; +} + +$opt{insert} ||= 1 unless $opt{update} || $opt{'insert-update'}; + +use RT; +use RT::Interface::CLI qw(GetCurrentUser); + +if ($opt{config}) { + die "Can't find configuration file $opt{config}" unless -f $opt{config}; + no warnings 'redefine'; + require RT::Config; + my $old = \&RT::Config::Configs; + *RT::Config::Configs = sub { return ($opt{config}, $old->(@_)) }; +} + +RT->LoadConfig(); +RT->Config->Set( LogToSTDERR => $opt{debug} ? 'debug' : 'warning' ); +RT->Config->Set( DateDayBeforeMonth => 1 ) if $opt{dmy}; +RT->Config->Set( DateDayBeforeMonth => 0 ) if $opt{mdy}; +RT->Config->Set( AutoCreateNonExternalUsers => 0 ); +RT->Init(); + +RT::Logger->add_callback( + sub { + my %p = @_; + return $p{message} unless $RT::Extension::Import::CSV::CurrentRow && $RT::Extension::Import::CSV::CurrentLine; + + my $row_line = "[row $RT::Extension::Import::CSV::CurrentRow, line $RT::Extension::Import::CSV::CurrentLine]"; + + # append the create message with the custom field values + # we need to be able to output the values without modifying $ticket->Create itself. + if ( $p{level} eq 'info' && $p{message} =~ /^Ticket \d+ created in queue/ ) { + no warnings 'once'; + return "$row_line $p{message}. $RT::Extension::Import::CSV::UniqueFields"; + } + + return "$row_line $p{message}"; + } +); + +use Test::MockTime; +require RT::Extension::Import::CSV; + +my $current_user; + +if ( ( $opt{type} eq 'transaction' ) && ( my $name = RT->Config->Get('TransactionsImportActor') ) ) { + $current_user = RT::CurrentUser->new( RT->SystemUser ); + $current_user->Load($name); + unless ( $current_user->Id ) { + RT->Logger->error("Couldn't find user $name"); + exit(1); + } +} +else { + $current_user = GetCurrentUser(); + unless ( $current_user->Id ) { + RT->Logger->error("No RT user found. Please consult your RT administrator."); + exit(1); + } +} + +my $timezone = $current_user->UserObj->Timezone; + +if ( !$timezone || $timezone ne 'UTC' ) { + my ( $ret, $msg ) = $current_user->UserObj->SetTimezone('UTC'); + if ( !$ret ) { + RT->Logger->error("Failed to temporarily set current user's timezone to UTC: $msg"); + exit(1); + } +} + +{ + use RT::Transaction; + no warnings 'redefine'; + my $orig_create = \&RT::Transaction::Create; + *RT::Transaction::Create = sub { + my $self = shift; + return $self->$orig_create( @_, $opt{'run-scrips'} ? () : ( ActivateScrips => 0 ) ); + }; +} + +{ + use RT::Record; + no warnings 'redefine'; + *RT::Record::AddCustomFieldDefaultValues = sub { return 1 }; +} + +my ( $created, $updated, $skipped, $skipped_ref ) = RT::Extension::Import::CSV->run( + CurrentUser => $current_user, + File => $file, + Type => $opt{type}, + Update => $opt{update}, + Insert => $opt{insert}, + InsertUpdate => $opt{'insert-update'}, + Force => $opt{force}, + ArticleClass => $opt{'article-class'}, +); + +print <<"EOF"; +created: $created +updated: $updated +skipped: $skipped +EOF + +# Write out skipped CSV data to a new file if we have any +if ( $skipped_ref ){ + my $filename = $file; + $filename =~ s/\.[a-z]{3}$//i; # remove file extension, if present + $filename .= "_skipped.csv"; + + my $csv_out = Text::CSV_XS->new( + { + binary => 1, + sep_char => ';', + quote_char => '`', + escape_char => '`', + always_quote => 1, + eol => $/, + %{ RT->Config->Get('CSVOptions') || {} }, + } + ); + + if ( scalar @$skipped_ref && open (my $fh, ">:encoding(utf8)", $filename) ){ + + foreach my $item ( @$skipped_ref ){ + $csv_out->print($fh, $item); + } + close $fh or RT::Logger->error("Unable to close file $filename: $!"); + print "\nSkipped rows written to file $filename\n"; + } + elsif ( scalar @$skipped_ref ){ + RT::Logger->error("Unable to create file $filename: $!"); + } +} + +if ( !$timezone || $timezone ne 'UTC' ) { + my ( $ret, $msg ) = $current_user->UserObj->SetTimezone( $timezone ); + if ( !$ret ) { + RT->Logger->error( "Failed to set current user's timezone back: $msg" ); + } +} + +__END__ + +=head1 NAME + +rt-extension-csv-importer - Import data and create tickets in RT + +=head1 SYNOPSIS + + rt-extension-csv-importer --config /path/to/config.pm --type user /path/to/user-data.csv + rt-extension-csv-importer --config /path/to/config.pm --type ticket /path/to/ticket-data.csv + rt-extension-csv-importer --config /path/to/config.pm --type ticket --update /path/to/ticket-data.csv + rt-extension-csv-importer --config /path/to/config.pm --type transaction /path/to/transaction-data.csv + rt-extension-csv-importer --config /path/to/config.pm --type article --article-class 'VM-Assessment' /path/to/article-data.csv + +=head1 DESCRIPTION + +This script will import/update RT from data in a CSV file. See +L for configuration. + +=head1 OPTIONS + +=over + +=item C<--config> I or C<-c> I + +Provides an explicit extra configuration file which is loaded before any +other configuration files. This is useful to provide per-import settings. +F should B contain settings for options with hashes +that may get merged with per-import settings. This option is required. + +=item C<--type> I or C<-t> I + +Specify which type of data we shall import + +=item C<--article-class> I
+ +Specify the article class when type is article. + +=item C<--update> + +Without this option, existing tickets (as determined by matching +defined values) are left untouched. With this option +provided, records will be updated based on their values in the CSV. +If an existing ticket is not found, the entry is skipped. Use the +C<--insert-update> option to create new tickets when they are not +found. + +=item C<--insert> + +By default, if C is not set, the script will +refuse to create tickets. Using this flag to create tickets in +that case. + +It's enabled automatically if C<--update> or C<--insert-update> +are not specified. + +=item C<--insert-update> + +This option will update a ticket if found or insert (create) a new +ticket if not found. + +=item C<--force> + +By default, tickets containing not-existing users will be skipped; with this +flag, they will be created without those users. + +This also allows the script to continue processing even errors are found +when parsing csv. + +=item C<--run-scrips> + +By default, scrips are temporarily deactivated on C creation. Use +this flag to allow RT to run scrips accordingly. + +=item C<--mdy>, C<--dmy> + +Force RT to parse dates as C or C
, respectively. In +the absence of this option, RT will default to the C +setting, which defaults to C
. + +=item C<--debug> + +Provide verbose output to STDERR during the import. + +=back diff --git a/inc/Module/Install.pm b/inc/Module/Install.pm new file mode 100644 index 0000000..7ba98c2 --- /dev/null +++ b/inc/Module/Install.pm @@ -0,0 +1,451 @@ +#line 1 +package Module::Install; + +# For any maintainers: +# The load order for Module::Install is a bit magic. +# It goes something like this... +# +# IF ( host has Module::Install installed, creating author mode ) { +# 1. Makefile.PL calls "use inc::Module::Install" +# 2. $INC{inc/Module/Install.pm} set to installed version of inc::Module::Install +# 3. The installed version of inc::Module::Install loads +# 4. inc::Module::Install calls "require Module::Install" +# 5. The ./inc/ version of Module::Install loads +# } ELSE { +# 1. Makefile.PL calls "use inc::Module::Install" +# 2. $INC{inc/Module/Install.pm} set to ./inc/ version of Module::Install +# 3. The ./inc/ version of Module::Install loads +# } + +use 5.006; +use strict 'vars'; +use Cwd (); +use File::Find (); +use File::Path (); + +use vars qw{$VERSION $MAIN}; +BEGIN { + # All Module::Install core packages now require synchronised versions. + # This will be used to ensure we don't accidentally load old or + # different versions of modules. + # This is not enforced yet, but will be some time in the next few + # releases once we can make sure it won't clash with custom + # Module::Install extensions. + $VERSION = '1.19'; + + # Storage for the pseudo-singleton + $MAIN = undef; + + *inc::Module::Install::VERSION = *VERSION; + @inc::Module::Install::ISA = __PACKAGE__; + +} + +sub import { + my $class = shift; + my $self = $class->new(@_); + my $who = $self->_caller; + + #------------------------------------------------------------- + # all of the following checks should be included in import(), + # to allow "eval 'require Module::Install; 1' to test + # installation of Module::Install. (RT #51267) + #------------------------------------------------------------- + + # Whether or not inc::Module::Install is actually loaded, the + # $INC{inc/Module/Install.pm} is what will still get set as long as + # the caller loaded module this in the documented manner. + # If not set, the caller may NOT have loaded the bundled version, and thus + # they may not have a MI version that works with the Makefile.PL. This would + # result in false errors or unexpected behaviour. And we don't want that. + my $file = join( '/', 'inc', split /::/, __PACKAGE__ ) . '.pm'; + unless ( $INC{$file} ) { die <<"END_DIE" } + +Please invoke ${\__PACKAGE__} with: + + use inc::${\__PACKAGE__}; + +not: + + use ${\__PACKAGE__}; + +END_DIE + + # This reportedly fixes a rare Win32 UTC file time issue, but + # as this is a non-cross-platform XS module not in the core, + # we shouldn't really depend on it. See RT #24194 for detail. + # (Also, this module only supports Perl 5.6 and above). + eval "use Win32::UTCFileTime" if $^O eq 'MSWin32' && $] >= 5.006; + + # If the script that is loading Module::Install is from the future, + # then make will detect this and cause it to re-run over and over + # again. This is bad. Rather than taking action to touch it (which + # is unreliable on some platforms and requires write permissions) + # for now we should catch this and refuse to run. + if ( -f $0 ) { + my $s = (stat($0))[9]; + + # If the modification time is only slightly in the future, + # sleep briefly to remove the problem. + my $a = $s - time; + if ( $a > 0 and $a < 5 ) { sleep 5 } + + # Too far in the future, throw an error. + my $t = time; + if ( $s > $t ) { die <<"END_DIE" } + +Your installer $0 has a modification time in the future ($s > $t). + +This is known to create infinite loops in make. + +Please correct this, then run $0 again. + +END_DIE + } + + + # Build.PL was formerly supported, but no longer is due to excessive + # difficulty in implementing every single feature twice. + if ( $0 =~ /Build.PL$/i ) { die <<"END_DIE" } + +Module::Install no longer supports Build.PL. + +It was impossible to maintain duel backends, and has been deprecated. + +Please remove all Build.PL files and only use the Makefile.PL installer. + +END_DIE + + #------------------------------------------------------------- + + # To save some more typing in Module::Install installers, every... + # use inc::Module::Install + # ...also acts as an implicit use strict. + $^H |= strict::bits(qw(refs subs vars)); + + #------------------------------------------------------------- + + unless ( -f $self->{file} ) { + foreach my $key (keys %INC) { + delete $INC{$key} if $key =~ /Module\/Install/; + } + + local $^W; + require "$self->{path}/$self->{dispatch}.pm"; + File::Path::mkpath("$self->{prefix}/$self->{author}"); + $self->{admin} = "$self->{name}::$self->{dispatch}"->new( _top => $self ); + $self->{admin}->init; + @_ = ($class, _self => $self); + goto &{"$self->{name}::import"}; + } + + local $^W; + *{"${who}::AUTOLOAD"} = $self->autoload; + $self->preload; + + # Unregister loader and worker packages so subdirs can use them again + delete $INC{'inc/Module/Install.pm'}; + delete $INC{'Module/Install.pm'}; + + # Save to the singleton + $MAIN = $self; + + return 1; +} + +sub autoload { + my $self = shift; + my $who = $self->_caller; + my $cwd = Cwd::getcwd(); + my $sym = "${who}::AUTOLOAD"; + $sym->{$cwd} = sub { + my $pwd = Cwd::getcwd(); + if ( my $code = $sym->{$pwd} ) { + # Delegate back to parent dirs + goto &$code unless $cwd eq $pwd; + } + unless ($$sym =~ s/([^:]+)$//) { + # XXX: it looks like we can't retrieve the missing function + # via $$sym (usually $main::AUTOLOAD) in this case. + # I'm still wondering if we should slurp Makefile.PL to + # get some context or not ... + my ($package, $file, $line) = caller; + die <<"EOT"; +Unknown function is found at $file line $line. +Execution of $file aborted due to runtime errors. + +If you're a contributor to a project, you may need to install +some Module::Install extensions from CPAN (or other repository). +If you're a user of a module, please contact the author. +EOT + } + my $method = $1; + if ( uc($method) eq $method ) { + # Do nothing + return; + } elsif ( $method =~ /^_/ and $self->can($method) ) { + # Dispatch to the root M:I class + return $self->$method(@_); + } + + # Dispatch to the appropriate plugin + unshift @_, ( $self, $1 ); + goto &{$self->can('call')}; + }; +} + +sub preload { + my $self = shift; + unless ( $self->{extensions} ) { + $self->load_extensions( + "$self->{prefix}/$self->{path}", $self + ); + } + + my @exts = @{$self->{extensions}}; + unless ( @exts ) { + @exts = $self->{admin}->load_all_extensions; + } + + my %seen; + foreach my $obj ( @exts ) { + while (my ($method, $glob) = each %{ref($obj) . '::'}) { + next unless $obj->can($method); + next if $method =~ /^_/; + next if $method eq uc($method); + $seen{$method}++; + } + } + + my $who = $self->_caller; + foreach my $name ( sort keys %seen ) { + local $^W; + *{"${who}::$name"} = sub { + ${"${who}::AUTOLOAD"} = "${who}::$name"; + goto &{"${who}::AUTOLOAD"}; + }; + } +} + +sub new { + my ($class, %args) = @_; + + delete $INC{'FindBin.pm'}; + { + # to suppress the redefine warning + local $SIG{__WARN__} = sub {}; + require FindBin; + } + + # ignore the prefix on extension modules built from top level. + my $base_path = Cwd::abs_path($FindBin::Bin); + unless ( Cwd::abs_path(Cwd::getcwd()) eq $base_path ) { + delete $args{prefix}; + } + return $args{_self} if $args{_self}; + + $base_path = VMS::Filespec::unixify($base_path) if $^O eq 'VMS'; + + $args{dispatch} ||= 'Admin'; + $args{prefix} ||= 'inc'; + $args{author} ||= ($^O eq 'VMS' ? '_author' : '.author'); + $args{bundle} ||= 'inc/BUNDLES'; + $args{base} ||= $base_path; + $class =~ s/^\Q$args{prefix}\E:://; + $args{name} ||= $class; + $args{version} ||= $class->VERSION; + unless ( $args{path} ) { + $args{path} = $args{name}; + $args{path} =~ s!::!/!g; + } + $args{file} ||= "$args{base}/$args{prefix}/$args{path}.pm"; + $args{wrote} = 0; + + bless( \%args, $class ); +} + +sub call { + my ($self, $method) = @_; + my $obj = $self->load($method) or return; + splice(@_, 0, 2, $obj); + goto &{$obj->can($method)}; +} + +sub load { + my ($self, $method) = @_; + + $self->load_extensions( + "$self->{prefix}/$self->{path}", $self + ) unless $self->{extensions}; + + foreach my $obj (@{$self->{extensions}}) { + return $obj if $obj->can($method); + } + + my $admin = $self->{admin} or die <<"END_DIE"; +The '$method' method does not exist in the '$self->{prefix}' path! +Please remove the '$self->{prefix}' directory and run $0 again to load it. +END_DIE + + my $obj = $admin->load($method, 1); + push @{$self->{extensions}}, $obj; + + $obj; +} + +sub load_extensions { + my ($self, $path, $top) = @_; + + my $should_reload = 0; + unless ( grep { ! ref $_ and lc $_ eq lc $self->{prefix} } @INC ) { + unshift @INC, $self->{prefix}; + $should_reload = 1; + } + + foreach my $rv ( $self->find_extensions($path) ) { + my ($file, $pkg) = @{$rv}; + next if $self->{pathnames}{$pkg}; + + local $@; + my $new = eval { local $^W; require $file; $pkg->can('new') }; + unless ( $new ) { + warn $@ if $@; + next; + } + $self->{pathnames}{$pkg} = + $should_reload ? delete $INC{$file} : $INC{$file}; + push @{$self->{extensions}}, &{$new}($pkg, _top => $top ); + } + + $self->{extensions} ||= []; +} + +sub find_extensions { + my ($self, $path) = @_; + + my @found; + File::Find::find( {no_chdir => 1, wanted => sub { + my $file = $File::Find::name; + return unless $file =~ m!^\Q$path\E/(.+)\.pm\Z!is; + my $subpath = $1; + return if lc($subpath) eq lc($self->{dispatch}); + + $file = "$self->{path}/$subpath.pm"; + my $pkg = "$self->{name}::$subpath"; + $pkg =~ s!/!::!g; + + # If we have a mixed-case package name, assume case has been preserved + # correctly. Otherwise, root through the file to locate the case-preserved + # version of the package name. + if ( $subpath eq lc($subpath) || $subpath eq uc($subpath) ) { + my $content = Module::Install::_read($File::Find::name); + my $in_pod = 0; + foreach ( split /\n/, $content ) { + $in_pod = 1 if /^=\w/; + $in_pod = 0 if /^=cut/; + next if ($in_pod || /^=cut/); # skip pod text + next if /^\s*#/; # and comments + if ( m/^\s*package\s+($pkg)\s*;/i ) { + $pkg = $1; + last; + } + } + } + + push @found, [ $file, $pkg ]; + }}, $path ) if -d $path; + + @found; +} + + + + + +##################################################################### +# Common Utility Functions + +sub _caller { + my $depth = 0; + my $call = caller($depth); + while ( $call eq __PACKAGE__ ) { + $depth++; + $call = caller($depth); + } + return $call; +} + +sub _read { + local *FH; + open( FH, '<', $_[0] ) or die "open($_[0]): $!"; + binmode FH; + my $string = do { local $/; }; + close FH or die "close($_[0]): $!"; + return $string; +} + +sub _readperl { + my $string = Module::Install::_read($_[0]); + $string =~ s/(?:\015{1,2}\012|\015|\012)/\n/sg; + $string =~ s/(\n)\n*__(?:DATA|END)__\b.*\z/$1/s; + $string =~ s/\n\n=\w+.+?\n\n=cut\b.+?\n+/\n\n/sg; + return $string; +} + +sub _readpod { + my $string = Module::Install::_read($_[0]); + $string =~ s/(?:\015{1,2}\012|\015|\012)/\n/sg; + return $string if $_[0] =~ /\.pod\z/; + $string =~ s/(^|\n=cut\b.+?\n+)[^=\s].+?\n(\n=\w+|\z)/$1$2/sg; + $string =~ s/\n*=pod\b[^\n]*\n+/\n\n/sg; + $string =~ s/\n*=cut\b[^\n]*\n+/\n\n/sg; + $string =~ s/^\n+//s; + return $string; +} + +sub _write { + local *FH; + open( FH, '>', $_[0] ) or die "open($_[0]): $!"; + binmode FH; + foreach ( 1 .. $#_ ) { + print FH $_[$_] or die "print($_[0]): $!"; + } + close FH or die "close($_[0]): $!"; +} + +# _version is for processing module versions (eg, 1.03_05) not +# Perl versions (eg, 5.8.1). +sub _version { + my $s = shift || 0; + my $d =()= $s =~ /(\.)/g; + if ( $d >= 2 ) { + # Normalise multipart versions + $s =~ s/(\.)(\d{1,3})/sprintf("$1%03d",$2)/eg; + } + $s =~ s/^(\d+)\.?//; + my $l = $1 || 0; + my @v = map { + $_ . '0' x (3 - length $_) + } $s =~ /(\d{1,3})\D?/g; + $l = $l . '.' . join '', @v if @v; + return $l + 0; +} + +sub _cmp { + _version($_[1]) <=> _version($_[2]); +} + +# Cloned from Params::Util::_CLASS +sub _CLASS { + ( + defined $_[0] + and + ! ref $_[0] + and + $_[0] =~ m/^[^\W\d]\w*(?:::\w+)*\z/s + ) ? $_[0] : undef; +} + +1; + +# Copyright 2008 - 2012 Adam Kennedy. diff --git a/inc/Module/Install/Base.pm b/inc/Module/Install/Base.pm new file mode 100644 index 0000000..9fa42c2 --- /dev/null +++ b/inc/Module/Install/Base.pm @@ -0,0 +1,83 @@ +#line 1 +package Module::Install::Base; + +use strict 'vars'; +use vars qw{$VERSION}; +BEGIN { + $VERSION = '1.19'; +} + +# Suspend handler for "redefined" warnings +BEGIN { + my $w = $SIG{__WARN__}; + $SIG{__WARN__} = sub { $w }; +} + +#line 42 + +sub new { + my $class = shift; + unless ( defined &{"${class}::call"} ) { + *{"${class}::call"} = sub { shift->_top->call(@_) }; + } + unless ( defined &{"${class}::load"} ) { + *{"${class}::load"} = sub { shift->_top->load(@_) }; + } + bless { @_ }, $class; +} + +#line 61 + +sub AUTOLOAD { + local $@; + my $func = eval { shift->_top->autoload } or return; + goto &$func; +} + +#line 75 + +sub _top { + $_[0]->{_top}; +} + +#line 90 + +sub admin { + $_[0]->_top->{admin} + or + Module::Install::Base::FakeAdmin->new; +} + +#line 106 + +sub is_admin { + ! $_[0]->admin->isa('Module::Install::Base::FakeAdmin'); +} + +sub DESTROY {} + +package Module::Install::Base::FakeAdmin; + +use vars qw{$VERSION}; +BEGIN { + $VERSION = $Module::Install::Base::VERSION; +} + +my $fake; + +sub new { + $fake ||= bless(\@_, $_[0]); +} + +sub AUTOLOAD {} + +sub DESTROY {} + +# Restore warning handler +BEGIN { + $SIG{__WARN__} = $SIG{__WARN__}->(); +} + +1; + +#line 159 diff --git a/inc/Module/Install/Can.pm b/inc/Module/Install/Can.pm new file mode 100644 index 0000000..d65c753 --- /dev/null +++ b/inc/Module/Install/Can.pm @@ -0,0 +1,163 @@ +#line 1 +package Module::Install::Can; + +use strict; +use Config (); +use ExtUtils::MakeMaker (); +use Module::Install::Base (); + +use vars qw{$VERSION @ISA $ISCORE}; +BEGIN { + $VERSION = '1.19'; + @ISA = 'Module::Install::Base'; + $ISCORE = 1; +} + +# check if we can load some module +### Upgrade this to not have to load the module if possible +sub can_use { + my ($self, $mod, $ver) = @_; + $mod =~ s{::|\\}{/}g; + $mod .= '.pm' unless $mod =~ /\.pm$/i; + + my $pkg = $mod; + $pkg =~ s{/}{::}g; + $pkg =~ s{\.pm$}{}i; + + local $@; + eval { require $mod; $pkg->VERSION($ver || 0); 1 }; +} + +# Check if we can run some command +sub can_run { + my ($self, $cmd) = @_; + + my $_cmd = $cmd; + return $_cmd if (-x $_cmd or $_cmd = MM->maybe_command($_cmd)); + + for my $dir ((split /$Config::Config{path_sep}/, $ENV{PATH}), '.') { + next if $dir eq ''; + require File::Spec; + my $abs = File::Spec->catfile($dir, $cmd); + return $abs if (-x $abs or $abs = MM->maybe_command($abs)); + } + + return; +} + +# Can our C compiler environment build XS files +sub can_xs { + my $self = shift; + + # Ensure we have the CBuilder module + $self->configure_requires( 'ExtUtils::CBuilder' => 0.27 ); + + # Do we have the configure_requires checker? + local $@; + eval "require ExtUtils::CBuilder;"; + if ( $@ ) { + # They don't obey configure_requires, so it is + # someone old and delicate. Try to avoid hurting + # them by falling back to an older simpler test. + return $self->can_cc(); + } + + # Do we have a working C compiler + my $builder = ExtUtils::CBuilder->new( + quiet => 1, + ); + unless ( $builder->have_compiler ) { + # No working C compiler + return 0; + } + + # Write a C file representative of what XS becomes + require File::Temp; + my ( $FH, $tmpfile ) = File::Temp::tempfile( + "compilexs-XXXXX", + SUFFIX => '.c', + ); + binmode $FH; + print $FH <<'END_C'; +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" + +int main(int argc, char **argv) { + return 0; +} + +int boot_sanexs() { + return 1; +} + +END_C + close $FH; + + # Can the C compiler access the same headers XS does + my @libs = (); + my $object = undef; + eval { + local $^W = 0; + $object = $builder->compile( + source => $tmpfile, + ); + @libs = $builder->link( + objects => $object, + module_name => 'sanexs', + ); + }; + my $result = $@ ? 0 : 1; + + # Clean up all the build files + foreach ( $tmpfile, $object, @libs ) { + next unless defined $_; + 1 while unlink; + } + + return $result; +} + +# Can we locate a (the) C compiler +sub can_cc { + my $self = shift; + + if ($^O eq 'VMS') { + require ExtUtils::CBuilder; + my $builder = ExtUtils::CBuilder->new( + quiet => 1, + ); + return $builder->have_compiler; + } + + my @chunks = split(/ /, $Config::Config{cc}) or return; + + # $Config{cc} may contain args; try to find out the program part + while (@chunks) { + return $self->can_run("@chunks") || (pop(@chunks), next); + } + + return; +} + +# Fix Cygwin bug on maybe_command(); +if ( $^O eq 'cygwin' ) { + require ExtUtils::MM_Cygwin; + require ExtUtils::MM_Win32; + if ( ! defined(&ExtUtils::MM_Cygwin::maybe_command) ) { + *ExtUtils::MM_Cygwin::maybe_command = sub { + my ($self, $file) = @_; + if ($file =~ m{^/cygdrive/}i and ExtUtils::MM_Win32->can('maybe_command')) { + ExtUtils::MM_Win32->maybe_command($file); + } else { + ExtUtils::MM_Unix->maybe_command($file); + } + } + } +} + +1; + +__END__ + +#line 245 diff --git a/inc/Module/Install/Fetch.pm b/inc/Module/Install/Fetch.pm new file mode 100644 index 0000000..3072b08 --- /dev/null +++ b/inc/Module/Install/Fetch.pm @@ -0,0 +1,93 @@ +#line 1 +package Module::Install::Fetch; + +use strict; +use Module::Install::Base (); + +use vars qw{$VERSION @ISA $ISCORE}; +BEGIN { + $VERSION = '1.19'; + @ISA = 'Module::Install::Base'; + $ISCORE = 1; +} + +sub get_file { + my ($self, %args) = @_; + my ($scheme, $host, $path, $file) = + $args{url} =~ m|^(\w+)://([^/]+)(.+)/(.+)| or return; + + if ( $scheme eq 'http' and ! eval { require LWP::Simple; 1 } ) { + $args{url} = $args{ftp_url} + or (warn("LWP support unavailable!\n"), return); + ($scheme, $host, $path, $file) = + $args{url} =~ m|^(\w+)://([^/]+)(.+)/(.+)| or return; + } + + $|++; + print "Fetching '$file' from $host... "; + + unless (eval { require Socket; Socket::inet_aton($host) }) { + warn "'$host' resolve failed!\n"; + return; + } + + return unless $scheme eq 'ftp' or $scheme eq 'http'; + + require Cwd; + my $dir = Cwd::getcwd(); + chdir $args{local_dir} or return if exists $args{local_dir}; + + if (eval { require LWP::Simple; 1 }) { + LWP::Simple::mirror($args{url}, $file); + } + elsif (eval { require Net::FTP; 1 }) { eval { + # use Net::FTP to get past firewall + my $ftp = Net::FTP->new($host, Passive => 1, Timeout => 600); + $ftp->login("anonymous", 'anonymous@example.com'); + $ftp->cwd($path); + $ftp->binary; + $ftp->get($file) or (warn("$!\n"), return); + $ftp->quit; + } } + elsif (my $ftp = $self->can_run('ftp')) { eval { + # no Net::FTP, fallback to ftp.exe + require FileHandle; + my $fh = FileHandle->new; + + local $SIG{CHLD} = 'IGNORE'; + unless ($fh->open("|$ftp -n")) { + warn "Couldn't open ftp: $!\n"; + chdir $dir; return; + } + + my @dialog = split(/\n/, <<"END_FTP"); +open $host +user anonymous anonymous\@example.com +cd $path +binary +get $file $file +quit +END_FTP + foreach (@dialog) { $fh->print("$_\n") } + $fh->close; + } } + else { + warn "No working 'ftp' program available!\n"; + chdir $dir; return; + } + + unless (-f $file) { + warn "Fetching failed: $@\n"; + chdir $dir; return; + } + + return if exists $args{size} and -s $file != $args{size}; + system($args{run}) if exists $args{run}; + unlink($file) if $args{remove}; + + print(((!exists $args{check_for} or -e $args{check_for}) + ? "done!" : "failed! ($!)"), "\n"); + chdir $dir; return !$?; +} + +1; diff --git a/inc/Module/Install/Include.pm b/inc/Module/Install/Include.pm new file mode 100644 index 0000000..13fdcd0 --- /dev/null +++ b/inc/Module/Install/Include.pm @@ -0,0 +1,34 @@ +#line 1 +package Module::Install::Include; + +use strict; +use Module::Install::Base (); + +use vars qw{$VERSION @ISA $ISCORE}; +BEGIN { + $VERSION = '1.19'; + @ISA = 'Module::Install::Base'; + $ISCORE = 1; +} + +sub include { + shift()->admin->include(@_); +} + +sub include_deps { + shift()->admin->include_deps(@_); +} + +sub auto_include { + shift()->admin->auto_include(@_); +} + +sub auto_include_deps { + shift()->admin->auto_include_deps(@_); +} + +sub auto_include_dependent_dists { + shift()->admin->auto_include_dependent_dists(@_); +} + +1; diff --git a/inc/Module/Install/Makefile.pm b/inc/Module/Install/Makefile.pm new file mode 100644 index 0000000..13a4464 --- /dev/null +++ b/inc/Module/Install/Makefile.pm @@ -0,0 +1,418 @@ +#line 1 +package Module::Install::Makefile; + +use strict 'vars'; +use ExtUtils::MakeMaker (); +use Module::Install::Base (); +use Fcntl qw/:flock :seek/; + +use vars qw{$VERSION @ISA $ISCORE}; +BEGIN { + $VERSION = '1.19'; + @ISA = 'Module::Install::Base'; + $ISCORE = 1; +} + +sub Makefile { $_[0] } + +my %seen = (); + +sub prompt { + shift; + + # Infinite loop protection + my @c = caller(); + if ( ++$seen{"$c[1]|$c[2]|$_[0]"} > 3 ) { + die "Caught an potential prompt infinite loop ($c[1]|$c[2]|$_[0])"; + } + + # In automated testing or non-interactive session, always use defaults + if ( ($ENV{AUTOMATED_TESTING} or -! -t STDIN) and ! $ENV{PERL_MM_USE_DEFAULT} ) { + local $ENV{PERL_MM_USE_DEFAULT} = 1; + goto &ExtUtils::MakeMaker::prompt; + } else { + goto &ExtUtils::MakeMaker::prompt; + } +} + +# Store a cleaned up version of the MakeMaker version, +# since we need to behave differently in a variety of +# ways based on the MM version. +my $makemaker = eval $ExtUtils::MakeMaker::VERSION; + +# If we are passed a param, do a "newer than" comparison. +# Otherwise, just return the MakeMaker version. +sub makemaker { + ( @_ < 2 or $makemaker >= eval($_[1]) ) ? $makemaker : 0 +} + +# Ripped from ExtUtils::MakeMaker 6.56, and slightly modified +# as we only need to know here whether the attribute is an array +# or a hash or something else (which may or may not be appendable). +my %makemaker_argtype = ( + C => 'ARRAY', + CONFIG => 'ARRAY', +# CONFIGURE => 'CODE', # ignore + DIR => 'ARRAY', + DL_FUNCS => 'HASH', + DL_VARS => 'ARRAY', + EXCLUDE_EXT => 'ARRAY', + EXE_FILES => 'ARRAY', + FUNCLIST => 'ARRAY', + H => 'ARRAY', + IMPORTS => 'HASH', + INCLUDE_EXT => 'ARRAY', + LIBS => 'ARRAY', # ignore '' + MAN1PODS => 'HASH', + MAN3PODS => 'HASH', + META_ADD => 'HASH', + META_MERGE => 'HASH', + PL_FILES => 'HASH', + PM => 'HASH', + PMLIBDIRS => 'ARRAY', + PMLIBPARENTDIRS => 'ARRAY', + PREREQ_PM => 'HASH', + CONFIGURE_REQUIRES => 'HASH', + SKIP => 'ARRAY', + TYPEMAPS => 'ARRAY', + XS => 'HASH', +# VERSION => ['version',''], # ignore +# _KEEP_AFTER_FLUSH => '', + + clean => 'HASH', + depend => 'HASH', + dist => 'HASH', + dynamic_lib=> 'HASH', + linkext => 'HASH', + macro => 'HASH', + postamble => 'HASH', + realclean => 'HASH', + test => 'HASH', + tool_autosplit => 'HASH', + + # special cases where you can use makemaker_append + CCFLAGS => 'APPENDABLE', + DEFINE => 'APPENDABLE', + INC => 'APPENDABLE', + LDDLFLAGS => 'APPENDABLE', + LDFROM => 'APPENDABLE', +); + +sub makemaker_args { + my ($self, %new_args) = @_; + my $args = ( $self->{makemaker_args} ||= {} ); + foreach my $key (keys %new_args) { + if ($makemaker_argtype{$key}) { + if ($makemaker_argtype{$key} eq 'ARRAY') { + $args->{$key} = [] unless defined $args->{$key}; + unless (ref $args->{$key} eq 'ARRAY') { + $args->{$key} = [$args->{$key}] + } + push @{$args->{$key}}, + ref $new_args{$key} eq 'ARRAY' + ? @{$new_args{$key}} + : $new_args{$key}; + } + elsif ($makemaker_argtype{$key} eq 'HASH') { + $args->{$key} = {} unless defined $args->{$key}; + foreach my $skey (keys %{ $new_args{$key} }) { + $args->{$key}{$skey} = $new_args{$key}{$skey}; + } + } + elsif ($makemaker_argtype{$key} eq 'APPENDABLE') { + $self->makemaker_append($key => $new_args{$key}); + } + } + else { + if (defined $args->{$key}) { + warn qq{MakeMaker attribute "$key" is overriden; use "makemaker_append" to append values\n}; + } + $args->{$key} = $new_args{$key}; + } + } + return $args; +} + +# For mm args that take multiple space-separated args, +# append an argument to the current list. +sub makemaker_append { + my $self = shift; + my $name = shift; + my $args = $self->makemaker_args; + $args->{$name} = defined $args->{$name} + ? join( ' ', $args->{$name}, @_ ) + : join( ' ', @_ ); +} + +sub build_subdirs { + my $self = shift; + my $subdirs = $self->makemaker_args->{DIR} ||= []; + for my $subdir (@_) { + push @$subdirs, $subdir; + } +} + +sub clean_files { + my $self = shift; + my $clean = $self->makemaker_args->{clean} ||= {}; + %$clean = ( + %$clean, + FILES => join ' ', grep { length $_ } ($clean->{FILES} || (), @_), + ); +} + +sub realclean_files { + my $self = shift; + my $realclean = $self->makemaker_args->{realclean} ||= {}; + %$realclean = ( + %$realclean, + FILES => join ' ', grep { length $_ } ($realclean->{FILES} || (), @_), + ); +} + +sub libs { + my $self = shift; + my $libs = ref $_[0] ? shift : [ shift ]; + $self->makemaker_args( LIBS => $libs ); +} + +sub inc { + my $self = shift; + $self->makemaker_args( INC => shift ); +} + +sub _wanted_t { +} + +sub tests_recursive { + my $self = shift; + my $dir = shift || 't'; + unless ( -d $dir ) { + die "tests_recursive dir '$dir' does not exist"; + } + my %tests = map { $_ => 1 } split / /, ($self->tests || ''); + require File::Find; + File::Find::find( + sub { /\.t$/ and -f $_ and $tests{"$File::Find::dir/*.t"} = 1 }, + $dir + ); + $self->tests( join ' ', sort keys %tests ); +} + +sub write { + my $self = shift; + die "&Makefile->write() takes no arguments\n" if @_; + + # Check the current Perl version + my $perl_version = $self->perl_version; + if ( $perl_version ) { + eval "use $perl_version; 1" + or die "ERROR: perl: Version $] is installed, " + . "but we need version >= $perl_version"; + } + + # Make sure we have a new enough MakeMaker + require ExtUtils::MakeMaker; + + if ( $perl_version and $self->_cmp($perl_version, '5.006') >= 0 ) { + # This previous attempted to inherit the version of + # ExtUtils::MakeMaker in use by the module author, but this + # was found to be untenable as some authors build releases + # using future dev versions of EU:MM that nobody else has. + # Instead, #toolchain suggests we use 6.59 which is the most + # stable version on CPAN at time of writing and is, to quote + # ribasushi, "not terminally fucked, > and tested enough". + # TODO: We will now need to maintain this over time to push + # the version up as new versions are released. + $self->build_requires( 'ExtUtils::MakeMaker' => 6.59 ); + $self->configure_requires( 'ExtUtils::MakeMaker' => 6.59 ); + } else { + # Allow legacy-compatibility with 5.005 by depending on the + # most recent EU:MM that supported 5.005. + $self->build_requires( 'ExtUtils::MakeMaker' => 6.36 ); + $self->configure_requires( 'ExtUtils::MakeMaker' => 6.36 ); + } + + # Generate the MakeMaker params + my $args = $self->makemaker_args; + $args->{DISTNAME} = $self->name; + $args->{NAME} = $self->module_name || $self->name; + $args->{NAME} =~ s/-/::/g; + $args->{VERSION} = $self->version or die <<'EOT'; +ERROR: Can't determine distribution version. Please specify it +explicitly via 'version' in Makefile.PL, or set a valid $VERSION +in a module, and provide its file path via 'version_from' (or +'all_from' if you prefer) in Makefile.PL. +EOT + + if ( $self->tests ) { + my @tests = split ' ', $self->tests; + my %seen; + $args->{test} = { + TESTS => (join ' ', grep {!$seen{$_}++} @tests), + }; + } elsif ( $Module::Install::ExtraTests::use_extratests ) { + # Module::Install::ExtraTests doesn't set $self->tests and does its own tests via harness. + # So, just ignore our xt tests here. + } elsif ( -d 'xt' and ($Module::Install::AUTHOR or $ENV{RELEASE_TESTING}) ) { + $args->{test} = { + TESTS => join( ' ', map { "$_/*.t" } grep { -d $_ } qw{ t xt } ), + }; + } + if ( $] >= 5.005 ) { + $args->{ABSTRACT} = $self->abstract; + $args->{AUTHOR} = join ', ', @{$self->author || []}; + } + if ( $self->makemaker(6.10) ) { + $args->{NO_META} = 1; + #$args->{NO_MYMETA} = 1; + } + if ( $self->makemaker(6.17) and $self->sign ) { + $args->{SIGN} = 1; + } + unless ( $self->is_admin ) { + delete $args->{SIGN}; + } + if ( $self->makemaker(6.31) and $self->license ) { + $args->{LICENSE} = $self->license; + } + + my $prereq = ($args->{PREREQ_PM} ||= {}); + %$prereq = ( %$prereq, + map { @$_ } # flatten [module => version] + map { @$_ } + grep $_, + ($self->requires) + ); + + # Remove any reference to perl, PREREQ_PM doesn't support it + delete $args->{PREREQ_PM}->{perl}; + + # Merge both kinds of requires into BUILD_REQUIRES + my $build_prereq = ($args->{BUILD_REQUIRES} ||= {}); + %$build_prereq = ( %$build_prereq, + map { @$_ } # flatten [module => version] + map { @$_ } + grep $_, + ($self->configure_requires, $self->build_requires) + ); + + # Remove any reference to perl, BUILD_REQUIRES doesn't support it + delete $args->{BUILD_REQUIRES}->{perl}; + + # Delete bundled dists from prereq_pm, add it to Makefile DIR + my $subdirs = ($args->{DIR} || []); + if ($self->bundles) { + my %processed; + foreach my $bundle (@{ $self->bundles }) { + my ($mod_name, $dist_dir) = @$bundle; + delete $prereq->{$mod_name}; + $dist_dir = File::Basename::basename($dist_dir); # dir for building this module + if (not exists $processed{$dist_dir}) { + if (-d $dist_dir) { + # List as sub-directory to be processed by make + push @$subdirs, $dist_dir; + } + # Else do nothing: the module is already present on the system + $processed{$dist_dir} = undef; + } + } + } + + unless ( $self->makemaker('6.55_03') ) { + %$prereq = (%$prereq,%$build_prereq); + delete $args->{BUILD_REQUIRES}; + } + + if ( my $perl_version = $self->perl_version ) { + eval "use $perl_version; 1" + or die "ERROR: perl: Version $] is installed, " + . "but we need version >= $perl_version"; + + if ( $self->makemaker(6.48) ) { + $args->{MIN_PERL_VERSION} = $perl_version; + } + } + + if ($self->installdirs) { + warn qq{old INSTALLDIRS (probably set by makemaker_args) is overriden by installdirs\n} if $args->{INSTALLDIRS}; + $args->{INSTALLDIRS} = $self->installdirs; + } + + my %args = map { + ( $_ => $args->{$_} ) } grep {defined($args->{$_} ) + } keys %$args; + + my $user_preop = delete $args{dist}->{PREOP}; + if ( my $preop = $self->admin->preop($user_preop) ) { + foreach my $key ( keys %$preop ) { + $args{dist}->{$key} = $preop->{$key}; + } + } + + my $mm = ExtUtils::MakeMaker::WriteMakefile(%args); + $self->fix_up_makefile($mm->{FIRST_MAKEFILE} || 'Makefile'); +} + +sub fix_up_makefile { + my $self = shift; + my $makefile_name = shift; + my $top_class = ref($self->_top) || ''; + my $top_version = $self->_top->VERSION || ''; + + my $preamble = $self->preamble + ? "# Preamble by $top_class $top_version\n" + . $self->preamble + : ''; + my $postamble = "# Postamble by $top_class $top_version\n" + . ($self->postamble || ''); + + local *MAKEFILE; + open MAKEFILE, "+< $makefile_name" or die "fix_up_makefile: Couldn't open $makefile_name: $!"; + eval { flock MAKEFILE, LOCK_EX }; + my $makefile = do { local $/; }; + + $makefile =~ s/\b(test_harness\(\$\(TEST_VERBOSE\), )/$1'inc', /; + $makefile =~ s/( -I\$\(INST_ARCHLIB\))/ -Iinc$1/g; + $makefile =~ s/( "-I\$\(INST_LIB\)")/ "-Iinc"$1/g; + $makefile =~ s/^(FULLPERL = .*)/$1 "-Iinc"/m; + $makefile =~ s/^(PERL = .*)/$1 "-Iinc"/m; + + # Module::Install will never be used to build the Core Perl + # Sometimes PERL_LIB and PERL_ARCHLIB get written anyway, which breaks + # PREFIX/PERL5LIB, and thus, install_share. Blank them if they exist + $makefile =~ s/^PERL_LIB = .+/PERL_LIB =/m; + #$makefile =~ s/^PERL_ARCHLIB = .+/PERL_ARCHLIB =/m; + + # Perl 5.005 mentions PERL_LIB explicitly, so we have to remove that as well. + $makefile =~ s/(\"?)-I\$\(PERL_LIB\)\1//g; + + # XXX - This is currently unused; not sure if it breaks other MM-users + # $makefile =~ s/^pm_to_blib\s+:\s+/pm_to_blib :: /mg; + + seek MAKEFILE, 0, SEEK_SET; + truncate MAKEFILE, 0; + print MAKEFILE "$preamble$makefile$postamble" or die $!; + close MAKEFILE or die $!; + + 1; +} + +sub preamble { + my ($self, $text) = @_; + $self->{preamble} = $text . $self->{preamble} if defined $text; + $self->{preamble}; +} + +sub postamble { + my ($self, $text) = @_; + $self->{postamble} ||= $self->admin->postamble; + $self->{postamble} .= $text if defined $text; + $self->{postamble} +} + +1; + +__END__ + +#line 544 diff --git a/inc/Module/Install/Metadata.pm b/inc/Module/Install/Metadata.pm new file mode 100644 index 0000000..11bf971 --- /dev/null +++ b/inc/Module/Install/Metadata.pm @@ -0,0 +1,722 @@ +#line 1 +package Module::Install::Metadata; + +use strict 'vars'; +use Module::Install::Base (); + +use vars qw{$VERSION @ISA $ISCORE}; +BEGIN { + $VERSION = '1.19'; + @ISA = 'Module::Install::Base'; + $ISCORE = 1; +} + +my @boolean_keys = qw{ + sign +}; + +my @scalar_keys = qw{ + name + module_name + abstract + version + distribution_type + tests + installdirs +}; + +my @tuple_keys = qw{ + configure_requires + build_requires + requires + recommends + bundles + resources +}; + +my @resource_keys = qw{ + homepage + bugtracker + repository +}; + +my @array_keys = qw{ + keywords + author +}; + +*authors = \&author; + +sub Meta { shift } +sub Meta_BooleanKeys { @boolean_keys } +sub Meta_ScalarKeys { @scalar_keys } +sub Meta_TupleKeys { @tuple_keys } +sub Meta_ResourceKeys { @resource_keys } +sub Meta_ArrayKeys { @array_keys } + +foreach my $key ( @boolean_keys ) { + *$key = sub { + my $self = shift; + if ( defined wantarray and not @_ ) { + return $self->{values}->{$key}; + } + $self->{values}->{$key} = ( @_ ? $_[0] : 1 ); + return $self; + }; +} + +foreach my $key ( @scalar_keys ) { + *$key = sub { + my $self = shift; + return $self->{values}->{$key} if defined wantarray and !@_; + $self->{values}->{$key} = shift; + return $self; + }; +} + +foreach my $key ( @array_keys ) { + *$key = sub { + my $self = shift; + return $self->{values}->{$key} if defined wantarray and !@_; + $self->{values}->{$key} ||= []; + push @{$self->{values}->{$key}}, @_; + return $self; + }; +} + +foreach my $key ( @resource_keys ) { + *$key = sub { + my $self = shift; + unless ( @_ ) { + return () unless $self->{values}->{resources}; + return map { $_->[1] } + grep { $_->[0] eq $key } + @{ $self->{values}->{resources} }; + } + return $self->{values}->{resources}->{$key} unless @_; + my $uri = shift or die( + "Did not provide a value to $key()" + ); + $self->resources( $key => $uri ); + return 1; + }; +} + +foreach my $key ( grep { $_ ne "resources" } @tuple_keys) { + *$key = sub { + my $self = shift; + return $self->{values}->{$key} unless @_; + my @added; + while ( @_ ) { + my $module = shift or last; + my $version = shift || 0; + push @added, [ $module, $version ]; + } + push @{ $self->{values}->{$key} }, @added; + return map {@$_} @added; + }; +} + +# Resource handling +my %lc_resource = map { $_ => 1 } qw{ + homepage + license + bugtracker + repository +}; + +sub resources { + my $self = shift; + while ( @_ ) { + my $name = shift or last; + my $value = shift or next; + if ( $name eq lc $name and ! $lc_resource{$name} ) { + die("Unsupported reserved lowercase resource '$name'"); + } + $self->{values}->{resources} ||= []; + push @{ $self->{values}->{resources} }, [ $name, $value ]; + } + $self->{values}->{resources}; +} + +# Aliases for build_requires that will have alternative +# meanings in some future version of META.yml. +sub test_requires { shift->build_requires(@_) } +sub install_requires { shift->build_requires(@_) } + +# Aliases for installdirs options +sub install_as_core { $_[0]->installdirs('perl') } +sub install_as_cpan { $_[0]->installdirs('site') } +sub install_as_site { $_[0]->installdirs('site') } +sub install_as_vendor { $_[0]->installdirs('vendor') } + +sub dynamic_config { + my $self = shift; + my $value = @_ ? shift : 1; + if ( $self->{values}->{dynamic_config} ) { + # Once dynamic we never change to static, for safety + return 0; + } + $self->{values}->{dynamic_config} = $value ? 1 : 0; + return 1; +} + +# Convenience command +sub static_config { + shift->dynamic_config(0); +} + +sub perl_version { + my $self = shift; + return $self->{values}->{perl_version} unless @_; + my $version = shift or die( + "Did not provide a value to perl_version()" + ); + + # Normalize the version + $version = $self->_perl_version($version); + + # We don't support the really old versions + unless ( $version >= 5.005 ) { + die "Module::Install only supports 5.005 or newer (use ExtUtils::MakeMaker)\n"; + } + + $self->{values}->{perl_version} = $version; +} + +sub all_from { + my ( $self, $file ) = @_; + + unless ( defined($file) ) { + my $name = $self->name or die( + "all_from called with no args without setting name() first" + ); + $file = join('/', 'lib', split(/-/, $name)) . '.pm'; + $file =~ s{.*/}{} unless -e $file; + unless ( -e $file ) { + die("all_from cannot find $file from $name"); + } + } + unless ( -f $file ) { + die("The path '$file' does not exist, or is not a file"); + } + + $self->{values}{all_from} = $file; + + # Some methods pull from POD instead of code. + # If there is a matching .pod, use that instead + my $pod = $file; + $pod =~ s/\.pm$/.pod/i; + $pod = $file unless -e $pod; + + # Pull the different values + $self->name_from($file) unless $self->name; + $self->version_from($file) unless $self->version; + $self->perl_version_from($file) unless $self->perl_version; + $self->author_from($pod) unless @{$self->author || []}; + $self->license_from($pod) unless $self->license; + $self->abstract_from($pod) unless $self->abstract; + + return 1; +} + +sub provides { + my $self = shift; + my $provides = ( $self->{values}->{provides} ||= {} ); + %$provides = (%$provides, @_) if @_; + return $provides; +} + +sub auto_provides { + my $self = shift; + return $self unless $self->is_admin; + unless (-e 'MANIFEST') { + warn "Cannot deduce auto_provides without a MANIFEST, skipping\n"; + return $self; + } + # Avoid spurious warnings as we are not checking manifest here. + local $SIG{__WARN__} = sub {1}; + require ExtUtils::Manifest; + local *ExtUtils::Manifest::manicheck = sub { return }; + + require Module::Build; + my $build = Module::Build->new( + dist_name => $self->name, + dist_version => $self->version, + license => $self->license, + ); + $self->provides( %{ $build->find_dist_packages || {} } ); +} + +sub feature { + my $self = shift; + my $name = shift; + my $features = ( $self->{values}->{features} ||= [] ); + my $mods; + + if ( @_ == 1 and ref( $_[0] ) ) { + # The user used ->feature like ->features by passing in the second + # argument as a reference. Accomodate for that. + $mods = $_[0]; + } else { + $mods = \@_; + } + + my $count = 0; + push @$features, ( + $name => [ + map { + ref($_) ? ( ref($_) eq 'HASH' ) ? %$_ : @$_ : $_ + } @$mods + ] + ); + + return @$features; +} + +sub features { + my $self = shift; + while ( my ( $name, $mods ) = splice( @_, 0, 2 ) ) { + $self->feature( $name, @$mods ); + } + return $self->{values}->{features} + ? @{ $self->{values}->{features} } + : (); +} + +sub no_index { + my $self = shift; + my $type = shift; + push @{ $self->{values}->{no_index}->{$type} }, @_ if $type; + return $self->{values}->{no_index}; +} + +sub read { + my $self = shift; + $self->include_deps( 'YAML::Tiny', 0 ); + + require YAML::Tiny; + my $data = YAML::Tiny::LoadFile('META.yml'); + + # Call methods explicitly in case user has already set some values. + while ( my ( $key, $value ) = each %$data ) { + next unless $self->can($key); + if ( ref $value eq 'HASH' ) { + while ( my ( $module, $version ) = each %$value ) { + $self->can($key)->($self, $module => $version ); + } + } else { + $self->can($key)->($self, $value); + } + } + return $self; +} + +sub write { + my $self = shift; + return $self unless $self->is_admin; + $self->admin->write_meta; + return $self; +} + +sub version_from { + require ExtUtils::MM_Unix; + my ( $self, $file ) = @_; + $self->version( ExtUtils::MM_Unix->parse_version($file) ); + + # for version integrity check + $self->makemaker_args( VERSION_FROM => $file ); +} + +sub abstract_from { + require ExtUtils::MM_Unix; + my ( $self, $file ) = @_; + $self->abstract( + bless( + { DISTNAME => $self->name }, + 'ExtUtils::MM_Unix' + )->parse_abstract($file) + ); +} + +# Add both distribution and module name +sub name_from { + my ($self, $file) = @_; + if ( + Module::Install::_read($file) =~ m/ + ^ \s* + package \s* + ([\w:]+) + [\s|;]* + /ixms + ) { + my ($name, $module_name) = ($1, $1); + $name =~ s{::}{-}g; + $self->name($name); + unless ( $self->module_name ) { + $self->module_name($module_name); + } + } else { + die("Cannot determine name from $file\n"); + } +} + +sub _extract_perl_version { + if ( + $_[0] =~ m/ + ^\s* + (?:use|require) \s* + v? + ([\d_\.]+) + \s* ; + /ixms + ) { + my $perl_version = $1; + $perl_version =~ s{_}{}g; + return $perl_version; + } else { + return; + } +} + +sub perl_version_from { + my $self = shift; + my $perl_version=_extract_perl_version(Module::Install::_read($_[0])); + if ($perl_version) { + $self->perl_version($perl_version); + } else { + warn "Cannot determine perl version info from $_[0]\n"; + return; + } +} + +sub author_from { + my $self = shift; + my $content = Module::Install::_read($_[0]); + if ($content =~ m/ + =head \d \s+ (?:authors?)\b \s* + ([^\n]*) + | + =head \d \s+ (?:licen[cs]e|licensing|copyright|legal)\b \s* + .*? copyright .*? \d\d\d[\d.]+ \s* (?:\bby\b)? \s* + ([^\n]*) + /ixms) { + my $author = $1 || $2; + + # XXX: ugly but should work anyway... + if (eval "require Pod::Escapes; 1") { + # Pod::Escapes has a mapping table. + # It's in core of perl >= 5.9.3, and should be installed + # as one of the Pod::Simple's prereqs, which is a prereq + # of Pod::Text 3.x (see also below). + $author =~ s{ E<( (\d+) | ([A-Za-z]+) )> } + { + defined $2 + ? chr($2) + : defined $Pod::Escapes::Name2character_number{$1} + ? chr($Pod::Escapes::Name2character_number{$1}) + : do { + warn "Unknown escape: E<$1>"; + "E<$1>"; + }; + }gex; + } + elsif (eval "require Pod::Text; 1" && $Pod::Text::VERSION < 3) { + # Pod::Text < 3.0 has yet another mapping table, + # though the table name of 2.x and 1.x are different. + # (1.x is in core of Perl < 5.6, 2.x is in core of + # Perl < 5.9.3) + my $mapping = ($Pod::Text::VERSION < 2) + ? \%Pod::Text::HTML_Escapes + : \%Pod::Text::ESCAPES; + $author =~ s{ E<( (\d+) | ([A-Za-z]+) )> } + { + defined $2 + ? chr($2) + : defined $mapping->{$1} + ? $mapping->{$1} + : do { + warn "Unknown escape: E<$1>"; + "E<$1>"; + }; + }gex; + } + else { + $author =~ s{E}{<}g; + $author =~ s{E}{>}g; + } + $self->author($author); + } else { + warn "Cannot determine author info from $_[0]\n"; + } +} + +#Stolen from M::B +my %license_urls = ( + perl => 'http://dev.perl.org/licenses/', + apache => 'http://apache.org/licenses/LICENSE-2.0', + apache_1_1 => 'http://apache.org/licenses/LICENSE-1.1', + artistic => 'http://opensource.org/licenses/artistic-license.php', + artistic_2 => 'http://opensource.org/licenses/artistic-license-2.0.php', + lgpl => 'http://opensource.org/licenses/lgpl-license.php', + lgpl2 => 'http://opensource.org/licenses/lgpl-2.1.php', + lgpl3 => 'http://opensource.org/licenses/lgpl-3.0.html', + bsd => 'http://opensource.org/licenses/bsd-license.php', + gpl => 'http://opensource.org/licenses/gpl-license.php', + gpl2 => 'http://opensource.org/licenses/gpl-2.0.php', + gpl3 => 'http://opensource.org/licenses/gpl-3.0.html', + mit => 'http://opensource.org/licenses/mit-license.php', + mozilla => 'http://opensource.org/licenses/mozilla1.1.php', + open_source => undef, + unrestricted => undef, + restrictive => undef, + unknown => undef, +); + +sub license { + my $self = shift; + return $self->{values}->{license} unless @_; + my $license = shift or die( + 'Did not provide a value to license()' + ); + $license = __extract_license($license) || lc $license; + $self->{values}->{license} = $license; + + # Automatically fill in license URLs + if ( $license_urls{$license} ) { + $self->resources( license => $license_urls{$license} ); + } + + return 1; +} + +sub _extract_license { + my $pod = shift; + my $matched; + return __extract_license( + ($matched) = $pod =~ m/ + (=head \d \s+ L(?i:ICEN[CS]E|ICENSING)\b.*?) + (=head \d.*|=cut.*|)\z + /xms + ) || __extract_license( + ($matched) = $pod =~ m/ + (=head \d \s+ (?:C(?i:OPYRIGHTS?)|L(?i:EGAL))\b.*?) + (=head \d.*|=cut.*|)\z + /xms + ); +} + +sub __extract_license { + my $license_text = shift or return; + my @phrases = ( + '(?:under )?the same (?:terms|license) as (?:perl|the perl (?:\d )?programming language)' => 'perl', 1, + '(?:under )?the terms of (?:perl|the perl programming language) itself' => 'perl', 1, + 'Artistic and GPL' => 'perl', 1, + 'GNU general public license' => 'gpl', 1, + 'GNU public license' => 'gpl', 1, + 'GNU lesser general public license' => 'lgpl', 1, + 'GNU lesser public license' => 'lgpl', 1, + 'GNU library general public license' => 'lgpl', 1, + 'GNU library public license' => 'lgpl', 1, + 'GNU Free Documentation license' => 'unrestricted', 1, + 'GNU Affero General Public License' => 'open_source', 1, + '(?:Free)?BSD license' => 'bsd', 1, + 'Artistic license 2\.0' => 'artistic_2', 1, + 'Artistic license' => 'artistic', 1, + 'Apache (?:Software )?license' => 'apache', 1, + 'GPL' => 'gpl', 1, + 'LGPL' => 'lgpl', 1, + 'BSD' => 'bsd', 1, + 'Artistic' => 'artistic', 1, + 'MIT' => 'mit', 1, + 'Mozilla Public License' => 'mozilla', 1, + 'Q Public License' => 'open_source', 1, + 'OpenSSL License' => 'unrestricted', 1, + 'SSLeay License' => 'unrestricted', 1, + 'zlib License' => 'open_source', 1, + 'proprietary' => 'proprietary', 0, + ); + while ( my ($pattern, $license, $osi) = splice(@phrases, 0, 3) ) { + $pattern =~ s#\s+#\\s+#gs; + if ( $license_text =~ /\b$pattern\b/i ) { + return $license; + } + } + return ''; +} + +sub license_from { + my $self = shift; + if (my $license=_extract_license(Module::Install::_read($_[0]))) { + $self->license($license); + } else { + warn "Cannot determine license info from $_[0]\n"; + return 'unknown'; + } +} + +sub _extract_bugtracker { + my @links = $_[0] =~ m#L<( + https?\Q://rt.cpan.org/\E[^>]+| + https?\Q://github.com/\E[\w_]+/[\w_]+/issues| + https?\Q://code.google.com/p/\E[\w_\-]+/issues/list + )>#gx; + my %links; + @links{@links}=(); + @links=keys %links; + return @links; +} + +sub bugtracker_from { + my $self = shift; + my $content = Module::Install::_read($_[0]); + my @links = _extract_bugtracker($content); + unless ( @links ) { + warn "Cannot determine bugtracker info from $_[0]\n"; + return 0; + } + if ( @links > 1 ) { + warn "Found more than one bugtracker link in $_[0]\n"; + return 0; + } + + # Set the bugtracker + bugtracker( $links[0] ); + return 1; +} + +sub requires_from { + my $self = shift; + my $content = Module::Install::_readperl($_[0]); + my @requires = $content =~ m/^use\s+([^\W\d]\w*(?:::\w+)*)\s+(v?[\d\.]+)/mg; + while ( @requires ) { + my $module = shift @requires; + my $version = shift @requires; + $self->requires( $module => $version ); + } +} + +sub test_requires_from { + my $self = shift; + my $content = Module::Install::_readperl($_[0]); + my @requires = $content =~ m/^use\s+([^\W\d]\w*(?:::\w+)*)\s+([\d\.]+)/mg; + while ( @requires ) { + my $module = shift @requires; + my $version = shift @requires; + $self->test_requires( $module => $version ); + } +} + +# Convert triple-part versions (eg, 5.6.1 or 5.8.9) to +# numbers (eg, 5.006001 or 5.008009). +# Also, convert double-part versions (eg, 5.8) +sub _perl_version { + my $v = $_[-1]; + $v =~ s/^([1-9])\.([1-9]\d?\d?)$/sprintf("%d.%03d",$1,$2)/e; + $v =~ s/^([1-9])\.([1-9]\d?\d?)\.(0|[1-9]\d?\d?)$/sprintf("%d.%03d%03d",$1,$2,$3 || 0)/e; + $v =~ s/(\.\d\d\d)000$/$1/; + $v =~ s/_.+$//; + if ( ref($v) ) { + # Numify + $v = $v + 0; + } + return $v; +} + +sub add_metadata { + my $self = shift; + my %hash = @_; + for my $key (keys %hash) { + warn "add_metadata: $key is not prefixed with 'x_'.\n" . + "Use appopriate function to add non-private metadata.\n" unless $key =~ /^x_/; + $self->{values}->{$key} = $hash{$key}; + } +} + + +###################################################################### +# MYMETA Support + +sub WriteMyMeta { + die "WriteMyMeta has been deprecated"; +} + +sub write_mymeta_yaml { + my $self = shift; + + # We need YAML::Tiny to write the MYMETA.yml file + unless ( eval { require YAML::Tiny; 1; } ) { + return 1; + } + + # Generate the data + my $meta = $self->_write_mymeta_data or return 1; + + # Save as the MYMETA.yml file + print "Writing MYMETA.yml\n"; + YAML::Tiny::DumpFile('MYMETA.yml', $meta); +} + +sub write_mymeta_json { + my $self = shift; + + # We need JSON to write the MYMETA.json file + unless ( eval { require JSON; 1; } ) { + return 1; + } + + # Generate the data + my $meta = $self->_write_mymeta_data or return 1; + + # Save as the MYMETA.yml file + print "Writing MYMETA.json\n"; + Module::Install::_write( + 'MYMETA.json', + JSON->new->pretty(1)->canonical->encode($meta), + ); +} + +sub _write_mymeta_data { + my $self = shift; + + # If there's no existing META.yml there is nothing we can do + return undef unless -f 'META.yml'; + + # We need Parse::CPAN::Meta to load the file + unless ( eval { require Parse::CPAN::Meta; 1; } ) { + return undef; + } + + # Merge the perl version into the dependencies + my $val = $self->Meta->{values}; + my $perl = delete $val->{perl_version}; + if ( $perl ) { + $val->{requires} ||= []; + my $requires = $val->{requires}; + + # Canonize to three-dot version after Perl 5.6 + if ( $perl >= 5.006 ) { + $perl =~ s{^(\d+)\.(\d\d\d)(\d*)}{join('.', $1, int($2||0), int($3||0))}e + } + unshift @$requires, [ perl => $perl ]; + } + + # Load the advisory META.yml file + my @yaml = Parse::CPAN::Meta::LoadFile('META.yml'); + my $meta = $yaml[0]; + + # Overwrite the non-configure dependency hashes + delete $meta->{requires}; + delete $meta->{build_requires}; + delete $meta->{recommends}; + if ( exists $val->{requires} ) { + $meta->{requires} = { map { @$_ } @{ $val->{requires} } }; + } + if ( exists $val->{build_requires} ) { + $meta->{build_requires} = { map { @$_ } @{ $val->{build_requires} } }; + } + + return $meta; +} + +1; diff --git a/inc/Module/Install/RTx.pm b/inc/Module/Install/RTx.pm new file mode 100644 index 0000000..2dd9489 --- /dev/null +++ b/inc/Module/Install/RTx.pm @@ -0,0 +1,316 @@ +#line 1 +package Module::Install::RTx; + +use 5.008; +use strict; +use warnings; +no warnings 'once'; + +use Term::ANSIColor qw(:constants); +use Module::Install::Base; +use base 'Module::Install::Base'; +our $VERSION = '0.42'; + +use FindBin; +use File::Glob (); +use File::Basename (); + +my @DIRS = qw(etc lib html static bin sbin po var); +my @INDEX_DIRS = qw(lib bin sbin); + +sub RTx { + my ( $self, $name, $extra_args ) = @_; + $extra_args ||= {}; + + # Set up names + my $fname = $name; + $fname =~ s!-!/!g; + + $self->name( $name ) + unless $self->name; + $self->all_from( "lib/$fname.pm" ) + unless $self->version; + $self->abstract("$name Extension") + unless $self->abstract; + unless ( $extra_args->{no_readme_generation} ) { + $self->readme_from( "lib/$fname.pm", + { options => [ quotes => "none" ] } ); + } + $self->add_metadata("x_module_install_rtx_version", $VERSION ); + + my $installdirs = $ENV{INSTALLDIRS}; + for ( @ARGV ) { + if ( /INSTALLDIRS=(.*)/ ) { + $installdirs = $1; + } + } + + # Try to find RT.pm + my @prefixes = qw( /opt /usr/local /home /usr /sw /usr/share/request-tracker4); + $ENV{RTHOME} =~ s{/RT\.pm$}{} if defined $ENV{RTHOME}; + $ENV{RTHOME} =~ s{/lib/?$}{} if defined $ENV{RTHOME}; + my @try = $ENV{RTHOME} ? ($ENV{RTHOME}, "$ENV{RTHOME}/lib") : (); + while (1) { + my @look = @INC; + unshift @look, grep {defined and -d $_} @try; + push @look, grep {defined and -d $_} + map { ( "$_/rt5/lib", "$_/lib/rt5", "$_/rt4/lib", "$_/lib/rt4", "$_/lib" ) } @prefixes; + last if eval {local @INC = @look; require RT; $RT::LocalLibPath}; + + warn + "Cannot find the location of RT.pm that defines \$RT::LocalPath in: @look\n"; + my $given = $self->prompt("Path to directory containing your RT.pm:") or exit; + $given =~ s{/RT\.pm$}{}; + $given =~ s{/lib/?$}{}; + @try = ($given, "$given/lib"); + } + + print "Using RT configuration from $INC{'RT.pm'}:\n"; + + my $local_lib_path = $RT::LocalLibPath; + unshift @INC, $local_lib_path; + my $lib_path = File::Basename::dirname( $INC{'RT.pm'} ); + unshift @INC, $lib_path; + + # Set a baseline minimum version + unless ( $extra_args->{deprecated_rt} ) { + $self->requires_rt('4.0.0'); + } + + my $package = $name; + $package =~ s/-/::/g; + if ( $RT::CORED_PLUGINS{$package} ) { + my ($base_version) = $RT::VERSION =~ /(\d+\.\d+\.\d+)/; + die RED, <<"EOT"; + +**** Error: Your installed version of RT ($RT::VERSION) already + contains this extension in core, so you don't need to + install it. + + Check https://docs.bestpractical.com/rt/$base_version/RT_Config.html + to configure it. + +EOT + } + + # Installation locations + my %path; + my $plugin_path; + if ( $installdirs && $installdirs eq 'vendor' ) { + $plugin_path = $RT::PluginPath; + } else { + $plugin_path = $RT::LocalPluginPath; + } + $path{$_} = $plugin_path . "/$name/$_" + foreach @DIRS; + + # Copy RT 4.2.0 static files into NoAuth; insufficient for + # images, but good enough for css and js. + $path{static} = "$path{html}/NoAuth/" + unless $RT::StaticPath; + + # Delete the ones we don't need + delete $path{$_} for grep {not -d "$FindBin::Bin/$_"} keys %path; + + my %index = map { $_ => 1 } @INDEX_DIRS; + $self->no_index( directory => $_ ) foreach grep !$index{$_}, @DIRS; + + my $args = join ', ', map "q($_)", map { ($_, "\$(DESTDIR)$path{$_}") } + sort keys %path; + + printf "%-10s => %s\n", $_, $path{$_} for sort keys %path; + + if ( my @dirs = map { ( -D => $_ ) } grep $path{$_}, qw(bin html sbin etc) ) { + my @po = map { ( -o => $_ ) } + grep -f, + File::Glob::bsd_glob("po/*.po"); + $self->postamble(<< ".") if @po; +lexicons :: +\t\$(NOECHO) \$(PERL) -MLocale::Maketext::Extract::Run=xgettext -e \"xgettext(qw(@dirs @po))\" +. + } + + my $remove_files; + if( $extra_args->{'remove_files'} ){ + $self->include('Module::Install::RTx::Remove'); + our @remove_files; + eval { require "etc/upgrade/remove_files" } + or print "No remove file located, no files to remove\n"; + $remove_files = join ",", map {"q(\$(DESTDIR)$plugin_path/$name/$_)"} @remove_files; + } + + $self->include('Module::Install::RTx::Runtime') if $self->admin; + $self->include_deps( 'YAML::Tiny', 0 ) if $self->admin; + my $postamble = << "."; +install :: +\t\$(NOECHO) \$(PERL) -Ilib -I"$local_lib_path" -I"$lib_path" -Iinc -MModule::Install::RTx::Runtime -e"RTxPlugin()" +. + + if( $remove_files ){ + $postamble .= << "."; +\t\$(NOECHO) \$(PERL) -MModule::Install::RTx::Remove -e \"RTxRemove([$remove_files])\" +. + } + + $postamble .= << "."; +\t\$(NOECHO) \$(PERL) -MExtUtils::Install -e \"install({$args})\" +. + + if ( $path{var} and -d $RT::MasonDataDir ) { + my ( $uid, $gid ) = ( stat($RT::MasonDataDir) )[ 4, 5 ]; + $postamble .= << "."; +\t\$(NOECHO) chown -R $uid:$gid $path{var} +. + } + + my %has_etc; + if ( File::Glob::bsd_glob("$FindBin::Bin/etc/schema.*") ) { + $has_etc{schema}++; + } + if ( File::Glob::bsd_glob("$FindBin::Bin/etc/acl.*") ) { + $has_etc{acl}++; + } + if ( -e 'etc/initialdata' ) { $has_etc{initialdata}++; } + if ( grep { /\d+\.\d+\.\d+.*$/ } glob('etc/upgrade/*.*.*') ) { + $has_etc{upgrade}++; + } + + $self->postamble("$postamble\n"); + if ( $path{lib} ) { + $self->makemaker_args( INSTALLSITELIB => $path{'lib'} ); + $self->makemaker_args( INSTALLARCHLIB => $path{'lib'} ); + $self->makemaker_args( INSTALLVENDORLIB => $path{'lib'} ) + } else { + $self->makemaker_args( PM => { "" => "" }, ); + } + + $self->makemaker_args( INSTALLSITEMAN1DIR => "$RT::LocalPath/man/man1" ); + $self->makemaker_args( INSTALLSITEMAN3DIR => "$RT::LocalPath/man/man3" ); + $self->makemaker_args( INSTALLSITEARCH => "$RT::LocalPath/man" ); + + # INSTALLDIRS=vendor should install manpages into /usr/share/man. + # That is the default path in most distributions. Need input from + # Redhat, Centos etc. + $self->makemaker_args( INSTALLVENDORMAN1DIR => "/usr/share/man/man1" ); + $self->makemaker_args( INSTALLVENDORMAN3DIR => "/usr/share/man/man3" ); + $self->makemaker_args( INSTALLVENDORARCH => "/usr/share/man" ); + + if (%has_etc) { + print "For first-time installation, type 'make initdb'.\n"; + my $initdb = ''; + $initdb .= <<"." if $has_etc{schema}; +\t\$(NOECHO) \$(PERL) -Ilib -I"$local_lib_path" -I"$lib_path" -Iinc -MModule::Install::RTx::Runtime -e"RTxDatabase(qw(schema \$(NAME) \$(VERSION)))" +. + $initdb .= <<"." if $has_etc{acl}; +\t\$(NOECHO) \$(PERL) -Ilib -I"$local_lib_path" -I"$lib_path" -Iinc -MModule::Install::RTx::Runtime -e"RTxDatabase(qw(acl \$(NAME) \$(VERSION)))" +. + $initdb .= <<"." if $has_etc{initialdata}; +\t\$(NOECHO) \$(PERL) -Ilib -I"$local_lib_path" -I"$lib_path" -Iinc -MModule::Install::RTx::Runtime -e"RTxDatabase(qw(insert \$(NAME) \$(VERSION)))" +. + $self->postamble("initdb ::\n$initdb\n"); + $self->postamble("initialize-database ::\n$initdb\n"); + if ($has_etc{upgrade}) { + print "To upgrade from a previous version of this extension, use 'make upgrade-database'\n"; + my $upgradedb = qq|\t\$(NOECHO) \$(PERL) -Ilib -I"$local_lib_path" -I"$lib_path" -Iinc -MModule::Install::RTx::Runtime -e"RTxDatabase(qw(upgrade \$(NAME) \$(VERSION)))"\n|; + $self->postamble("upgrade-database ::\n$upgradedb\n"); + $self->postamble("upgradedb ::\n$upgradedb\n"); + } + } + +} + +sub requires_rt { + my ($self,$version) = @_; + + _load_rt_handle(); + + if ($self->is_admin) { + $self->add_metadata("x_requires_rt", $version); + my @sorted = sort RT::Handle::cmp_version $version,'4.0.0'; + $self->perl_version('5.008003') if $sorted[0] eq '4.0.0' + and (not $self->perl_version or '5.008003' > $self->perl_version); + @sorted = sort RT::Handle::cmp_version $version,'4.2.0'; + $self->perl_version('5.010001') if $sorted[0] eq '4.2.0' + and (not $self->perl_version or '5.010001' > $self->perl_version); + } + + # if we're exactly the same version as what we want, silently return + return if ($version eq $RT::VERSION); + + my @sorted = sort RT::Handle::cmp_version $version,$RT::VERSION; + + if ($sorted[-1] eq $version) { + die RED, <<"EOT"; + +**** Error: This extension requires RT $version. Your installed version + of RT ($RT::VERSION) is too old. + +EOT + } +} + +sub requires_rt_plugin { + my $self = shift; + my ( $plugin ) = @_; + + if ($self->is_admin) { + my $plugins = $self->Meta->{values}{"x_requires_rt_plugins"} || []; + push @{$plugins}, $plugin; + $self->add_metadata("x_requires_rt_plugins", $plugins); + } + + my $path = $plugin; + $path =~ s{\:\:}{-}g; + $path = "$RT::LocalPluginPath/$path/lib"; + if ( -e $path ) { + unshift @INC, $path; + } else { + my $name = $self->name; + my $msg = <<"EOT"; + +**** Warning: $name requires that the $plugin plugin be installed and + enabled; it does not appear to be installed. +EOT + warn RED, $msg, RESET, "\n"; + } + $self->requires(@_); +} + +sub rt_too_new { + my ($self,$version,$msg) = @_; + my $name = $self->name; + $msg ||= <add_metadata("x_rt_too_new", $version) if $self->is_admin; + + _load_rt_handle(); + my @sorted = sort RT::Handle::cmp_version $version,$RT::VERSION; + + if ($sorted[0] eq $version) { + warn RED, sprintf($msg,$RT::VERSION), RESET, "\n"; + } +} + +# RT::Handle runs FinalizeDatabaseType which calls RT->Config->Get +# On 3.8, this dies. On 4.0/4.2 ->Config transparently runs LoadConfig. +# LoadConfig requires being able to read RT_SiteConfig.pm (root) so we'd +# like to avoid pushing that on users. +# Fake up just enough Config to let FinalizeDatabaseType finish, and +# anyone later calling LoadConfig will overwrite our shenanigans. +sub _load_rt_handle { + unless ($RT::Config) { + require RT::Config; + $RT::Config = RT::Config->new; + RT->Config->Set('DatabaseType','mysql'); + } + require RT::Handle; +} + +1; + +__END__ + +#line 484 diff --git a/inc/Module/Install/RTx/Runtime.pm b/inc/Module/Install/RTx/Runtime.pm new file mode 100644 index 0000000..ae07502 --- /dev/null +++ b/inc/Module/Install/RTx/Runtime.pm @@ -0,0 +1,80 @@ +#line 1 +package Module::Install::RTx::Runtime; + +use base 'Exporter'; +our @EXPORT = qw/RTxDatabase RTxPlugin/; + +use strict; +use File::Basename (); + +sub _rt_runtime_load { + require RT; + + eval { RT::LoadConfig(); }; + if (my $err = $@) { + die $err unless $err =~ /^RT couldn't load RT config file/m; + my $warn = <can('AddUpgradeHistory'); + + my $lib_path = File::Basename::dirname($INC{'RT.pm'}); + my @args = ( + "-I.", + "-Ilib", + "-I$RT::LocalLibPath", + "-I$lib_path", + "$RT::SbinPath/rt-setup-database", + "--action" => $action, + ($action eq 'upgrade' ? () : ("--datadir" => "etc")), + (($action eq 'insert') ? ("--datafile" => "etc/initialdata") : ()), + "--dba" => $RT::DatabaseAdmin || $RT::DatabaseUser, + "--prompt-for-dba-password" => '', + ($has_upgrade ? ("--package" => $name, "--ext-version" => $version) : ()), + ); + # If we're upgrading against an RT which isn't at least 4.2 (has + # AddUpgradeHistory) then pass --package. Upgrades against later RT + # releases will pick up --package from AddUpgradeHistory. + if ($action eq 'upgrade' and not $has_upgrade) { + push @args, "--package" => $name; + } + + print "$^X @args\n"; + (system($^X, @args) == 0) or die "...returned with error: $?\n"; +} + +sub RTxPlugin { + my ($name) = @_; + + _rt_runtime_load(); + require YAML::Tiny; + my $data = YAML::Tiny::LoadFile('META.yml'); + my $name = $data->{name}; + + my @enabled = RT->Config->Get('Plugins'); + for my $required (@{$data->{x_requires_rt_plugins} || []}) { + next if grep {$required eq $_} @enabled; + + warn <<"EOT"; + +**** Warning: $name requires that the $required plugin be installed and + enabled; it is not currently in \@Plugins. + +EOT + } +} + +1; diff --git a/inc/Module/Install/ReadmeFromPod.pm b/inc/Module/Install/ReadmeFromPod.pm new file mode 100644 index 0000000..3738232 --- /dev/null +++ b/inc/Module/Install/ReadmeFromPod.pm @@ -0,0 +1,184 @@ +#line 1 +package Module::Install::ReadmeFromPod; + +use 5.006; +use strict; +use warnings; +use base qw(Module::Install::Base); +use vars qw($VERSION); + +$VERSION = '0.30'; + +{ + + # these aren't defined until after _require_admin is run, so + # define them so prototypes are available during compilation. + sub io; + sub capture(&;@); + +#line 28 + + my $done = 0; + + sub _require_admin { + + # do this once to avoid redefinition warnings from IO::All + return if $done; + + require IO::All; + IO::All->import( '-binary' ); + + require Capture::Tiny; + Capture::Tiny->import ( 'capture' ); + + return; + } + +} + +sub readme_from { + my $self = shift; + return unless $self->is_admin; + + _require_admin; + + # Input file + my $in_file = shift || $self->_all_from + or die "Can't determine file to make readme_from"; + + # Get optional arguments + my ($clean, $format, $out_file, $options); + my $args = shift; + if ( ref $args ) { + # Arguments are in a hashref + if ( ref($args) ne 'HASH' ) { + die "Expected a hashref but got a ".ref($args)."\n"; + } else { + $clean = $args->{'clean'}; + $format = $args->{'format'}; + $out_file = $args->{'output_file'}; + $options = $args->{'options'}; + } + } else { + # Arguments are in a list + $clean = $args; + $format = shift; + $out_file = shift; + $options = \@_; + } + + # Default values; + $clean ||= 0; + $format ||= 'txt'; + + # Generate README + print "readme_from $in_file to $format\n"; + if ($format =~ m/te?xt/) { + $out_file = $self->_readme_txt($in_file, $out_file, $options); + } elsif ($format =~ m/html?/) { + $out_file = $self->_readme_htm($in_file, $out_file, $options); + } elsif ($format eq 'man') { + $out_file = $self->_readme_man($in_file, $out_file, $options); + } elsif ($format eq 'md') { + $out_file = $self->_readme_md($in_file, $out_file, $options); + } elsif ($format eq 'pdf') { + $out_file = $self->_readme_pdf($in_file, $out_file, $options); + } + + if ($clean) { + $self->clean_files($out_file); + } + + return 1; +} + + +sub _readme_txt { + my ($self, $in_file, $out_file, $options) = @_; + $out_file ||= 'README'; + require Pod::Text; + my $parser = Pod::Text->new( @$options ); + my $io = io->file($out_file)->open(">"); + my $out_fh = $io->io_handle; + $parser->output_fh( *$out_fh ); + $parser->parse_file( $in_file ); + return $out_file; +} + + +sub _readme_htm { + my ($self, $in_file, $out_file, $options) = @_; + $out_file ||= 'README.htm'; + require Pod::Html; + my ($o) = capture { + Pod::Html::pod2html( + "--infile=$in_file", + "--outfile=-", + @$options, + ); + }; + io->file($out_file)->print($o); + # Remove temporary files if needed + for my $file ('pod2htmd.tmp', 'pod2htmi.tmp') { + if (-e $file) { + unlink $file or warn "Warning: Could not remove file '$file'.\n$!\n"; + } + } + return $out_file; +} + + +sub _readme_man { + my ($self, $in_file, $out_file, $options) = @_; + $out_file ||= 'README.1'; + require Pod::Man; + my $parser = Pod::Man->new( @$options ); + my $io = io->file($out_file)->open(">"); + my $out_fh = $io->io_handle; + $parser->output_fh( *$out_fh ); + $parser->parse_file( $in_file ); + return $out_file; +} + + +sub _readme_pdf { + my ($self, $in_file, $out_file, $options) = @_; + $out_file ||= 'README.pdf'; + eval { require App::pod2pdf; } + or die "Could not generate $out_file because pod2pdf could not be found\n"; + my $parser = App::pod2pdf->new( @$options ); + $parser->parse_from_file($in_file); + my ($o) = capture { $parser->output }; + io->file($out_file)->print($o); + return $out_file; +} + +sub _readme_md { + my ($self, $in_file, $out_file, $options) = @_; + $out_file ||= 'README.md'; + require Pod::Markdown; + my $parser = Pod::Markdown->new( @$options ); + my $io = io->file($out_file)->open(">"); + my $out_fh = $io->io_handle; + $parser->output_fh( *$out_fh ); + $parser->parse_file( $in_file ); + return $out_file; +} + + +sub _all_from { + my $self = shift; + return unless $self->admin->{extensions}; + my ($metadata) = grep { + ref($_) eq 'Module::Install::Metadata'; + } @{$self->admin->{extensions}}; + return unless $metadata; + return $metadata->{values}{all_from} || ''; +} + +'Readme!'; + +__END__ + +#line 316 + diff --git a/inc/Module/Install/Substitute.pm b/inc/Module/Install/Substitute.pm new file mode 100644 index 0000000..56af7fe --- /dev/null +++ b/inc/Module/Install/Substitute.pm @@ -0,0 +1,131 @@ +#line 1 +package Module::Install::Substitute; + +use strict; +use warnings; +use 5.008; # I don't care much about earlier versions + +use Module::Install::Base; +our @ISA = qw(Module::Install::Base); + +our $VERSION = '0.03'; + +require File::Temp; +require File::Spec; +require Cwd; + +#line 89 + +sub substitute +{ + my $self = shift; + $self->{__subst} = shift; + $self->{__option} = {}; + if( UNIVERSAL::isa( $_[0], 'HASH' ) ) { + my $opts = shift; + while( my ($k,$v) = each( %$opts ) ) { + $self->{__option}->{ lc( $k ) } = $v || ''; + } + } + $self->_parse_options; + + my @file = @_; + foreach my $f (@file) { + $self->_rewrite_file( $f ); + } + + return; +} + +sub _parse_options +{ + my $self = shift; + my $cwd = Cwd::getcwd(); + foreach my $t ( qw(from to) ) { + $self->{__option}->{$t} = $cwd unless $self->{__option}->{$t}; + my $d = $self->{__option}->{$t}; + die "Couldn't read directory '$d'" unless -d $d && -r _; + } +} + +sub _rewrite_file +{ + my ($self, $file) = @_; + my $source = File::Spec->catfile( $self->{__option}{from}, $file ); + $source .= $self->{__option}{sufix} if $self->{__option}{sufix}; + unless( -f $source && -r _ ) { + print STDERR "Couldn't find file '$source'\n"; + return; + } + my $dest = File::Spec->catfile( $self->{__option}{to}, $file ); + return $self->__rewrite_file( $source, $dest ); +} + +sub __rewrite_file +{ + my ($self, $source, $dest) = @_; + + my $mode = (stat($source))[2]; + + open my $sfh, "<$source" or die "Couldn't open '$source' for read"; + print "Open input '$source' file for substitution\n"; + + my ($tmpfh, $tmpfname) = File::Temp::tempfile('mi-subst-XXXX', UNLINK => 1); + $self->__process_streams( $sfh, $tmpfh, ($source eq $dest)? 1: 0 ); + close $sfh; + + seek $tmpfh, 0, 0 or die "Couldn't seek in tmp file"; + + open my $dfh, ">$dest" or die "Couldn't open '$dest' for write"; + print "Open output '$dest' file for substitution\n"; + + while( <$tmpfh> ) { + print $dfh $_; + } + close $dfh; + chmod $mode, $dest or "Couldn't change mode on '$dest'"; +} + +sub __process_streams +{ + my ($self, $in, $out, $replace) = @_; + + my @queue = (); + my $subst = $self->{'__subst'}; + my $re_subst = join('|', map {"\Q$_"} keys %{ $subst } ); + + while( my $str = <$in> ) { + if( $str =~ /^###\s*(before|replace|after)\:\s?(.*)$/s ) { + my ($action, $nstr) = ($1,$2); + $nstr =~ s/\@($re_subst)\@/$subst->{$1}/ge; + + die "Replace action is bad idea for situations when dest is equal to source" + if $replace && $action eq 'replace'; + if( $action eq 'before' ) { + die "no line before 'before' action" unless @queue; + # overwrite prev line; + pop @queue; + push @queue, $nstr; + push @queue, $str; + } elsif( $action eq 'replace' ) { + push @queue, $nstr; + } elsif( $action eq 'after' ) { + push @queue, $str; + push @queue, $nstr; + # skip one line; + <$in>; + } + } else { + push @queue, $str; + } + while( @queue > 3 ) { + print $out shift(@queue); + } + } + while( scalar @queue ) { + print $out shift(@queue); + } +} + +1; + diff --git a/inc/Module/Install/Win32.pm b/inc/Module/Install/Win32.pm new file mode 100644 index 0000000..f7aa615 --- /dev/null +++ b/inc/Module/Install/Win32.pm @@ -0,0 +1,64 @@ +#line 1 +package Module::Install::Win32; + +use strict; +use Module::Install::Base (); + +use vars qw{$VERSION @ISA $ISCORE}; +BEGIN { + $VERSION = '1.19'; + @ISA = 'Module::Install::Base'; + $ISCORE = 1; +} + +# determine if the user needs nmake, and download it if needed +sub check_nmake { + my $self = shift; + $self->load('can_run'); + $self->load('get_file'); + + require Config; + return unless ( + $^O eq 'MSWin32' and + $Config::Config{make} and + $Config::Config{make} =~ /^nmake\b/i and + ! $self->can_run('nmake') + ); + + print "The required 'nmake' executable not found, fetching it...\n"; + + require File::Basename; + my $rv = $self->get_file( + url => 'http://download.microsoft.com/download/vc15/Patch/1.52/W95/EN-US/Nmake15.exe', + ftp_url => 'ftp://ftp.microsoft.com/Softlib/MSLFILES/Nmake15.exe', + local_dir => File::Basename::dirname($^X), + size => 51928, + run => 'Nmake15.exe /o > nul', + check_for => 'Nmake.exe', + remove => 1, + ); + + die <<'END_MESSAGE' unless $rv; + +------------------------------------------------------------------------------- + +Since you are using Microsoft Windows, you will need the 'nmake' utility +before installation. It's available at: + + http://download.microsoft.com/download/vc15/Patch/1.52/W95/EN-US/Nmake15.exe + or + ftp://ftp.microsoft.com/Softlib/MSLFILES/Nmake15.exe + +Please download the file manually, save it to a directory in %PATH% (e.g. +C:\WINDOWS\COMMAND\), then launch the MS-DOS command line shell, "cd" to +that directory, and run "Nmake15.exe" from there; that will create the +'nmake.exe' file needed by this module. + +You may then resume the installation process described in README. + +------------------------------------------------------------------------------- +END_MESSAGE + +} + +1; diff --git a/inc/Module/Install/WriteAll.pm b/inc/Module/Install/WriteAll.pm new file mode 100644 index 0000000..2db861a --- /dev/null +++ b/inc/Module/Install/WriteAll.pm @@ -0,0 +1,63 @@ +#line 1 +package Module::Install::WriteAll; + +use strict; +use Module::Install::Base (); + +use vars qw{$VERSION @ISA $ISCORE}; +BEGIN { + $VERSION = '1.19'; + @ISA = qw{Module::Install::Base}; + $ISCORE = 1; +} + +sub WriteAll { + my $self = shift; + my %args = ( + meta => 1, + sign => 0, + inline => 0, + check_nmake => 1, + @_, + ); + + $self->sign(1) if $args{sign}; + $self->admin->WriteAll(%args) if $self->is_admin; + + $self->check_nmake if $args{check_nmake}; + unless ( $self->makemaker_args->{PL_FILES} ) { + # XXX: This still may be a bit over-defensive... + unless ($self->makemaker(6.25)) { + $self->makemaker_args( PL_FILES => {} ) if -f 'Build.PL'; + } + } + + # Until ExtUtils::MakeMaker support MYMETA.yml, make sure + # we clean it up properly ourself. + $self->realclean_files('MYMETA.yml'); + + if ( $args{inline} ) { + $self->Inline->write; + } else { + $self->Makefile->write; + } + + # The Makefile write process adds a couple of dependencies, + # so write the META.yml files after the Makefile. + if ( $args{meta} ) { + $self->Meta->write; + } + + # Experimental support for MYMETA + if ( $ENV{X_MYMETA} ) { + if ( $ENV{X_MYMETA} eq 'JSON' ) { + $self->Meta->write_mymeta_json; + } else { + $self->Meta->write_mymeta_yaml; + } + } + + return 1; +} + +1; diff --git a/inc/YAML/Tiny.pm b/inc/YAML/Tiny.pm new file mode 100644 index 0000000..fb157a6 --- /dev/null +++ b/inc/YAML/Tiny.pm @@ -0,0 +1,872 @@ +#line 1 +use 5.008001; # sane UTF-8 support +use strict; +use warnings; +package YAML::Tiny; # git description: v1.72-7-g8682f63 +# XXX-INGY is 5.8.1 too old/broken for utf8? +# XXX-XDG Lancaster consensus was that it was sufficient until +# proven otherwise + +our $VERSION = '1.73'; + +##################################################################### +# The YAML::Tiny API. +# +# These are the currently documented API functions/methods and +# exports: + +use Exporter; +our @ISA = qw{ Exporter }; +our @EXPORT = qw{ Load Dump }; +our @EXPORT_OK = qw{ LoadFile DumpFile freeze thaw }; + +### +# Functional/Export API: + +sub Dump { + return YAML::Tiny->new(@_)->_dump_string; +} + +# XXX-INGY Returning last document seems a bad behavior. +# XXX-XDG I think first would seem more natural, but I don't know +# that it's worth changing now +sub Load { + my $self = YAML::Tiny->_load_string(@_); + if ( wantarray ) { + return @$self; + } else { + # To match YAML.pm, return the last document + return $self->[-1]; + } +} + +# XXX-INGY Do we really need freeze and thaw? +# XXX-XDG I don't think so. I'd support deprecating them. +BEGIN { + *freeze = \&Dump; + *thaw = \&Load; +} + +sub DumpFile { + my $file = shift; + return YAML::Tiny->new(@_)->_dump_file($file); +} + +sub LoadFile { + my $file = shift; + my $self = YAML::Tiny->_load_file($file); + if ( wantarray ) { + return @$self; + } else { + # Return only the last document to match YAML.pm, + return $self->[-1]; + } +} + + +### +# Object Oriented API: + +# Create an empty YAML::Tiny object +# XXX-INGY Why do we use ARRAY object? +# NOTE: I get it now, but I think it's confusing and not needed. +# Will change it on a branch later, for review. +# +# XXX-XDG I don't support changing it yet. It's a very well-documented +# "API" of YAML::Tiny. I'd support deprecating it, but Adam suggested +# we not change it until YAML.pm's own OO API is established so that +# users only have one API change to digest, not two +sub new { + my $class = shift; + bless [ @_ ], $class; +} + +# XXX-INGY It probably doesn't matter, and it's probably too late to +# change, but 'read/write' are the wrong names. Read and Write +# are actions that take data from storage to memory +# characters/strings. These take the data to/from storage to native +# Perl objects, which the terms dump and load are meant. As long as +# this is a legacy quirk to YAML::Tiny it's ok, but I'd prefer not +# to add new {read,write}_* methods to this API. + +sub read_string { + my $self = shift; + $self->_load_string(@_); +} + +sub write_string { + my $self = shift; + $self->_dump_string(@_); +} + +sub read { + my $self = shift; + $self->_load_file(@_); +} + +sub write { + my $self = shift; + $self->_dump_file(@_); +} + + + + +##################################################################### +# Constants + +# Printed form of the unprintable characters in the lowest range +# of ASCII characters, listed by ASCII ordinal position. +my @UNPRINTABLE = qw( + 0 x01 x02 x03 x04 x05 x06 a + b t n v f r x0E x0F + x10 x11 x12 x13 x14 x15 x16 x17 + x18 x19 x1A e x1C x1D x1E x1F +); + +# Printable characters for escapes +my %UNESCAPES = ( + 0 => "\x00", z => "\x00", N => "\x85", + a => "\x07", b => "\x08", t => "\x09", + n => "\x0a", v => "\x0b", f => "\x0c", + r => "\x0d", e => "\x1b", '\\' => '\\', +); + +# XXX-INGY +# I(ngy) need to decide if these values should be quoted in +# YAML::Tiny or not. Probably yes. + +# These 3 values have special meaning when unquoted and using the +# default YAML schema. They need quotes if they are strings. +my %QUOTE = map { $_ => 1 } qw{ + null true false +}; + +# The commented out form is simpler, but overloaded the Perl regex +# engine due to recursion and backtracking problems on strings +# larger than 32,000ish characters. Keep it for reference purposes. +# qr/\"((?:\\.|[^\"])*)\"/ +my $re_capture_double_quoted = qr/\"([^\\"]*(?:\\.[^\\"]*)*)\"/; +my $re_capture_single_quoted = qr/\'([^\']*(?:\'\'[^\']*)*)\'/; +# unquoted re gets trailing space that needs to be stripped +my $re_capture_unquoted_key = qr/([^:]+(?::+\S(?:[^:]*|.*?(?=:)))*)(?=\s*\:(?:\s+|$))/; +my $re_trailing_comment = qr/(?:\s+\#.*)?/; +my $re_key_value_separator = qr/\s*:(?:\s+(?:\#.*)?|$)/; + + + + + +##################################################################### +# YAML::Tiny Implementation. +# +# These are the private methods that do all the work. They may change +# at any time. + + +### +# Loader functions: + +# Create an object from a file +sub _load_file { + my $class = ref $_[0] ? ref shift : shift; + + # Check the file + my $file = shift or $class->_error( 'You did not specify a file name' ); + $class->_error( "File '$file' does not exist" ) + unless -e $file; + $class->_error( "'$file' is a directory, not a file" ) + unless -f _; + $class->_error( "Insufficient permissions to read '$file'" ) + unless -r _; + + # Open unbuffered with strict UTF-8 decoding and no translation layers + open( my $fh, "<:unix:encoding(UTF-8)", $file ); + unless ( $fh ) { + $class->_error("Failed to open file '$file': $!"); + } + + # flock if available (or warn if not possible for OS-specific reasons) + if ( _can_flock() ) { + flock( $fh, Fcntl::LOCK_SH() ) + or warn "Couldn't lock '$file' for reading: $!"; + } + + # slurp the contents + my $contents = eval { + use warnings FATAL => 'utf8'; + local $/; + <$fh> + }; + if ( my $err = $@ ) { + $class->_error("Error reading from file '$file': $err"); + } + + # close the file (release the lock) + unless ( close $fh ) { + $class->_error("Failed to close file '$file': $!"); + } + + $class->_load_string( $contents ); +} + +# Create an object from a string +sub _load_string { + my $class = ref $_[0] ? ref shift : shift; + my $self = bless [], $class; + my $string = $_[0]; + eval { + unless ( defined $string ) { + die \"Did not provide a string to load"; + } + + # Check if Perl has it marked as characters, but it's internally + # inconsistent. E.g. maybe latin1 got read on a :utf8 layer + if ( utf8::is_utf8($string) && ! utf8::valid($string) ) { + die \<<'...'; +Read an invalid UTF-8 string (maybe mixed UTF-8 and 8-bit character set). +Did you decode with lax ":utf8" instead of strict ":encoding(UTF-8)"? +... + } + + # Ensure Unicode character semantics, even for 0x80-0xff + utf8::upgrade($string); + + # Check for and strip any leading UTF-8 BOM + $string =~ s/^\x{FEFF}//; + + # Check for some special cases + return $self unless length $string; + + # Split the file into lines + my @lines = grep { ! /^\s*(?:\#.*)?\z/ } + split /(?:\015{1,2}\012|\015|\012)/, $string; + + # Strip the initial YAML header + @lines and $lines[0] =~ /^\%YAML[: ][\d\.]+.*\z/ and shift @lines; + + # A nibbling parser + my $in_document = 0; + while ( @lines ) { + # Do we have a document header? + if ( $lines[0] =~ /^---\s*(?:(.+)\s*)?\z/ ) { + # Handle scalar documents + shift @lines; + if ( defined $1 and $1 !~ /^(?:\#.+|\%YAML[: ][\d\.]+)\z/ ) { + push @$self, + $self->_load_scalar( "$1", [ undef ], \@lines ); + next; + } + $in_document = 1; + } + + if ( ! @lines or $lines[0] =~ /^(?:---|\.\.\.)/ ) { + # A naked document + push @$self, undef; + while ( @lines and $lines[0] !~ /^---/ ) { + shift @lines; + } + $in_document = 0; + + # XXX The final '-+$' is to look for -- which ends up being an + # error later. + } elsif ( ! $in_document && @$self ) { + # only the first document can be explicit + die \"YAML::Tiny failed to classify the line '$lines[0]'"; + } elsif ( $lines[0] =~ /^\s*\-(?:\s|$|-+$)/ ) { + # An array at the root + my $document = [ ]; + push @$self, $document; + $self->_load_array( $document, [ 0 ], \@lines ); + + } elsif ( $lines[0] =~ /^(\s*)\S/ ) { + # A hash at the root + my $document = { }; + push @$self, $document; + $self->_load_hash( $document, [ length($1) ], \@lines ); + + } else { + # Shouldn't get here. @lines have whitespace-only lines + # stripped, and previous match is a line with any + # non-whitespace. So this clause should only be reachable via + # a perlbug where \s is not symmetric with \S + + # uncoverable statement + die \"YAML::Tiny failed to classify the line '$lines[0]'"; + } + } + }; + my $err = $@; + if ( ref $err eq 'SCALAR' ) { + $self->_error(${$err}); + } elsif ( $err ) { + $self->_error($err); + } + + return $self; +} + +sub _unquote_single { + my ($self, $string) = @_; + return '' unless length $string; + $string =~ s/\'\'/\'/g; + return $string; +} + +sub _unquote_double { + my ($self, $string) = @_; + return '' unless length $string; + $string =~ s/\\"/"/g; + $string =~ + s{\\([Nnever\\fartz0b]|x([0-9a-fA-F]{2}))} + {(length($1)>1)?pack("H2",$2):$UNESCAPES{$1}}gex; + return $string; +} + +# Load a YAML scalar string to the actual Perl scalar +sub _load_scalar { + my ($self, $string, $indent, $lines) = @_; + + # Trim trailing whitespace + $string =~ s/\s*\z//; + + # Explitic null/undef + return undef if $string eq '~'; + + # Single quote + if ( $string =~ /^$re_capture_single_quoted$re_trailing_comment\z/ ) { + return $self->_unquote_single($1); + } + + # Double quote. + if ( $string =~ /^$re_capture_double_quoted$re_trailing_comment\z/ ) { + return $self->_unquote_double($1); + } + + # Special cases + if ( $string =~ /^[\'\"!&]/ ) { + die \"YAML::Tiny does not support a feature in line '$string'"; + } + return {} if $string =~ /^{}(?:\s+\#.*)?\z/; + return [] if $string =~ /^\[\](?:\s+\#.*)?\z/; + + # Regular unquoted string + if ( $string !~ /^[>|]/ ) { + die \"YAML::Tiny found illegal characters in plain scalar: '$string'" + if $string =~ /^(?:-(?:\s|$)|[\@\%\`])/ or + $string =~ /:(?:\s|$)/; + $string =~ s/\s+#.*\z//; + return $string; + } + + # Error + die \"YAML::Tiny failed to find multi-line scalar content" unless @$lines; + + # Check the indent depth + $lines->[0] =~ /^(\s*)/; + $indent->[-1] = length("$1"); + if ( defined $indent->[-2] and $indent->[-1] <= $indent->[-2] ) { + die \"YAML::Tiny found bad indenting in line '$lines->[0]'"; + } + + # Pull the lines + my @multiline = (); + while ( @$lines ) { + $lines->[0] =~ /^(\s*)/; + last unless length($1) >= $indent->[-1]; + push @multiline, substr(shift(@$lines), $indent->[-1]); + } + + my $j = (substr($string, 0, 1) eq '>') ? ' ' : "\n"; + my $t = (substr($string, 1, 1) eq '-') ? '' : "\n"; + return join( $j, @multiline ) . $t; +} + +# Load an array +sub _load_array { + my ($self, $array, $indent, $lines) = @_; + + while ( @$lines ) { + # Check for a new document + if ( $lines->[0] =~ /^(?:---|\.\.\.)/ ) { + while ( @$lines and $lines->[0] !~ /^---/ ) { + shift @$lines; + } + return 1; + } + + # Check the indent level + $lines->[0] =~ /^(\s*)/; + if ( length($1) < $indent->[-1] ) { + return 1; + } elsif ( length($1) > $indent->[-1] ) { + die \"YAML::Tiny found bad indenting in line '$lines->[0]'"; + } + + if ( $lines->[0] =~ /^(\s*\-\s+)[^\'\"]\S*\s*:(?:\s+|$)/ ) { + # Inline nested hash + my $indent2 = length("$1"); + $lines->[0] =~ s/-/ /; + push @$array, { }; + $self->_load_hash( $array->[-1], [ @$indent, $indent2 ], $lines ); + + } elsif ( $lines->[0] =~ /^\s*\-\s*\z/ ) { + shift @$lines; + unless ( @$lines ) { + push @$array, undef; + return 1; + } + if ( $lines->[0] =~ /^(\s*)\-/ ) { + my $indent2 = length("$1"); + if ( $indent->[-1] == $indent2 ) { + # Null array entry + push @$array, undef; + } else { + # Naked indenter + push @$array, [ ]; + $self->_load_array( + $array->[-1], [ @$indent, $indent2 ], $lines + ); + } + + } elsif ( $lines->[0] =~ /^(\s*)\S/ ) { + push @$array, { }; + $self->_load_hash( + $array->[-1], [ @$indent, length("$1") ], $lines + ); + + } else { + die \"YAML::Tiny failed to classify line '$lines->[0]'"; + } + + } elsif ( $lines->[0] =~ /^\s*\-(\s*)(.+?)\s*\z/ ) { + # Array entry with a value + shift @$lines; + push @$array, $self->_load_scalar( + "$2", [ @$indent, undef ], $lines + ); + + } elsif ( defined $indent->[-2] and $indent->[-1] == $indent->[-2] ) { + # This is probably a structure like the following... + # --- + # foo: + # - list + # bar: value + # + # ... so lets return and let the hash parser handle it + return 1; + + } else { + die \"YAML::Tiny failed to classify line '$lines->[0]'"; + } + } + + return 1; +} + +# Load a hash +sub _load_hash { + my ($self, $hash, $indent, $lines) = @_; + + while ( @$lines ) { + # Check for a new document + if ( $lines->[0] =~ /^(?:---|\.\.\.)/ ) { + while ( @$lines and $lines->[0] !~ /^---/ ) { + shift @$lines; + } + return 1; + } + + # Check the indent level + $lines->[0] =~ /^(\s*)/; + if ( length($1) < $indent->[-1] ) { + return 1; + } elsif ( length($1) > $indent->[-1] ) { + die \"YAML::Tiny found bad indenting in line '$lines->[0]'"; + } + + # Find the key + my $key; + + # Quoted keys + if ( $lines->[0] =~ + s/^\s*$re_capture_single_quoted$re_key_value_separator// + ) { + $key = $self->_unquote_single($1); + } + elsif ( $lines->[0] =~ + s/^\s*$re_capture_double_quoted$re_key_value_separator// + ) { + $key = $self->_unquote_double($1); + } + elsif ( $lines->[0] =~ + s/^\s*$re_capture_unquoted_key$re_key_value_separator// + ) { + $key = $1; + $key =~ s/\s+$//; + } + elsif ( $lines->[0] =~ /^\s*\?/ ) { + die \"YAML::Tiny does not support a feature in line '$lines->[0]'"; + } + else { + die \"YAML::Tiny failed to classify line '$lines->[0]'"; + } + + if ( exists $hash->{$key} ) { + warn "YAML::Tiny found a duplicate key '$key' in line '$lines->[0]'"; + } + + # Do we have a value? + if ( length $lines->[0] ) { + # Yes + $hash->{$key} = $self->_load_scalar( + shift(@$lines), [ @$indent, undef ], $lines + ); + } else { + # An indent + shift @$lines; + unless ( @$lines ) { + $hash->{$key} = undef; + return 1; + } + if ( $lines->[0] =~ /^(\s*)-/ ) { + $hash->{$key} = []; + $self->_load_array( + $hash->{$key}, [ @$indent, length($1) ], $lines + ); + } elsif ( $lines->[0] =~ /^(\s*)./ ) { + my $indent2 = length("$1"); + if ( $indent->[-1] >= $indent2 ) { + # Null hash entry + $hash->{$key} = undef; + } else { + $hash->{$key} = {}; + $self->_load_hash( + $hash->{$key}, [ @$indent, length($1) ], $lines + ); + } + } + } + } + + return 1; +} + + +### +# Dumper functions: + +# Save an object to a file +sub _dump_file { + my $self = shift; + + require Fcntl; + + # Check the file + my $file = shift or $self->_error( 'You did not specify a file name' ); + + my $fh; + # flock if available (or warn if not possible for OS-specific reasons) + if ( _can_flock() ) { + # Open without truncation (truncate comes after lock) + my $flags = Fcntl::O_WRONLY()|Fcntl::O_CREAT(); + sysopen( $fh, $file, $flags ) + or $self->_error("Failed to open file '$file' for writing: $!"); + + # Use no translation and strict UTF-8 + binmode( $fh, ":raw:encoding(UTF-8)"); + + flock( $fh, Fcntl::LOCK_EX() ) + or warn "Couldn't lock '$file' for reading: $!"; + + # truncate and spew contents + truncate $fh, 0; + seek $fh, 0, 0; + } + else { + open $fh, ">:unix:encoding(UTF-8)", $file; + } + + # serialize and spew to the handle + print {$fh} $self->_dump_string; + + # close the file (release the lock) + unless ( close $fh ) { + $self->_error("Failed to close file '$file': $!"); + } + + return 1; +} + +# Save an object to a string +sub _dump_string { + my $self = shift; + return '' unless ref $self && @$self; + + # Iterate over the documents + my $indent = 0; + my @lines = (); + + eval { + foreach my $cursor ( @$self ) { + push @lines, '---'; + + # An empty document + if ( ! defined $cursor ) { + # Do nothing + + # A scalar document + } elsif ( ! ref $cursor ) { + $lines[-1] .= ' ' . $self->_dump_scalar( $cursor ); + + # A list at the root + } elsif ( ref $cursor eq 'ARRAY' ) { + unless ( @$cursor ) { + $lines[-1] .= ' []'; + next; + } + push @lines, $self->_dump_array( $cursor, $indent, {} ); + + # A hash at the root + } elsif ( ref $cursor eq 'HASH' ) { + unless ( %$cursor ) { + $lines[-1] .= ' {}'; + next; + } + push @lines, $self->_dump_hash( $cursor, $indent, {} ); + + } else { + die \("Cannot serialize " . ref($cursor)); + } + } + }; + if ( ref $@ eq 'SCALAR' ) { + $self->_error(${$@}); + } elsif ( $@ ) { + $self->_error($@); + } + + join '', map { "$_\n" } @lines; +} + +sub _has_internal_string_value { + my $value = shift; + my $b_obj = B::svref_2object(\$value); # for round trip problem + return $b_obj->FLAGS & B::SVf_POK(); +} + +sub _dump_scalar { + my $string = $_[1]; + my $is_key = $_[2]; + # Check this before checking length or it winds up looking like a string! + my $has_string_flag = _has_internal_string_value($string); + return '~' unless defined $string; + return "''" unless length $string; + if (Scalar::Util::looks_like_number($string)) { + # keys and values that have been used as strings get quoted + if ( $is_key || $has_string_flag ) { + return qq['$string']; + } + else { + return $string; + } + } + if ( $string =~ /[\x00-\x09\x0b-\x0d\x0e-\x1f\x7f-\x9f\'\n]/ ) { + $string =~ s/\\/\\\\/g; + $string =~ s/"/\\"/g; + $string =~ s/\n/\\n/g; + $string =~ s/[\x85]/\\N/g; + $string =~ s/([\x00-\x1f])/\\$UNPRINTABLE[ord($1)]/g; + $string =~ s/([\x7f-\x9f])/'\x' . sprintf("%X",ord($1))/ge; + return qq|"$string"|; + } + if ( $string =~ /(?:^[~!@#%&*|>?:,'"`{}\[\]]|^-+$|\s|:\z)/ or + $QUOTE{$string} + ) { + return "'$string'"; + } + return $string; +} + +sub _dump_array { + my ($self, $array, $indent, $seen) = @_; + if ( $seen->{refaddr($array)}++ ) { + die \"YAML::Tiny does not support circular references"; + } + my @lines = (); + foreach my $el ( @$array ) { + my $line = (' ' x $indent) . '-'; + my $type = ref $el; + if ( ! $type ) { + $line .= ' ' . $self->_dump_scalar( $el ); + push @lines, $line; + + } elsif ( $type eq 'ARRAY' ) { + if ( @$el ) { + push @lines, $line; + push @lines, $self->_dump_array( $el, $indent + 1, $seen ); + } else { + $line .= ' []'; + push @lines, $line; + } + + } elsif ( $type eq 'HASH' ) { + if ( keys %$el ) { + push @lines, $line; + push @lines, $self->_dump_hash( $el, $indent + 1, $seen ); + } else { + $line .= ' {}'; + push @lines, $line; + } + + } else { + die \"YAML::Tiny does not support $type references"; + } + } + + @lines; +} + +sub _dump_hash { + my ($self, $hash, $indent, $seen) = @_; + if ( $seen->{refaddr($hash)}++ ) { + die \"YAML::Tiny does not support circular references"; + } + my @lines = (); + foreach my $name ( sort keys %$hash ) { + my $el = $hash->{$name}; + my $line = (' ' x $indent) . $self->_dump_scalar($name, 1) . ":"; + my $type = ref $el; + if ( ! $type ) { + $line .= ' ' . $self->_dump_scalar( $el ); + push @lines, $line; + + } elsif ( $type eq 'ARRAY' ) { + if ( @$el ) { + push @lines, $line; + push @lines, $self->_dump_array( $el, $indent + 1, $seen ); + } else { + $line .= ' []'; + push @lines, $line; + } + + } elsif ( $type eq 'HASH' ) { + if ( keys %$el ) { + push @lines, $line; + push @lines, $self->_dump_hash( $el, $indent + 1, $seen ); + } else { + $line .= ' {}'; + push @lines, $line; + } + + } else { + die \"YAML::Tiny does not support $type references"; + } + } + + @lines; +} + + + +##################################################################### +# DEPRECATED API methods: + +# Error storage (DEPRECATED as of 1.57) +our $errstr = ''; + +# Set error +sub _error { + require Carp; + $errstr = $_[1]; + $errstr =~ s/ at \S+ line \d+.*//; + Carp::croak( $errstr ); +} + +# Retrieve error +my $errstr_warned; +sub errstr { + require Carp; + Carp::carp( "YAML::Tiny->errstr and \$YAML::Tiny::errstr is deprecated" ) + unless $errstr_warned++; + $errstr; +} + + + + +##################################################################### +# Helper functions. Possibly not needed. + + +# Use to detect nv or iv +use B; + +# XXX-INGY Is flock YAML::Tiny's responsibility? +# Some platforms can't flock :-( +# XXX-XDG I think it is. When reading and writing files, we ought +# to be locking whenever possible. People (foolishly) use YAML +# files for things like session storage, which has race issues. +my $HAS_FLOCK; +sub _can_flock { + if ( defined $HAS_FLOCK ) { + return $HAS_FLOCK; + } + else { + require Config; + my $c = \%Config::Config; + $HAS_FLOCK = grep { $c->{$_} } qw/d_flock d_fcntl_can_lock d_lockf/; + require Fcntl if $HAS_FLOCK; + return $HAS_FLOCK; + } +} + + +# XXX-INGY Is this core in 5.8.1? Can we remove this? +# XXX-XDG Scalar::Util 1.18 didn't land until 5.8.8, so we need this +##################################################################### +# Use Scalar::Util if possible, otherwise emulate it + +use Scalar::Util (); +BEGIN { + local $@; + if ( eval { Scalar::Util->VERSION(1.18); } ) { + *refaddr = *Scalar::Util::refaddr; + } + else { + eval <<'END_PERL'; +# Scalar::Util failed to load or too old +sub refaddr { + my $pkg = ref($_[0]) or return undef; + if ( !! UNIVERSAL::can($_[0], 'can') ) { + bless $_[0], 'Scalar::Util::Fake'; + } else { + $pkg = undef; + } + "$_[0]" =~ /0x(\w+)/; + my $i = do { no warnings 'portable'; hex $1 }; + bless $_[0], $pkg if defined $pkg; + $i; +} +END_PERL + } +} + +delete $YAML::Tiny::{refaddr}; + +1; + +# XXX-INGY Doc notes I'm putting up here. Changing the doc when it's wrong +# but leaving grey area stuff up here. +# +# I would like to change Read/Write to Load/Dump below without +# changing the actual API names. +# +# It might be better to put Load/Dump API in the SYNOPSIS instead of the +# dubious OO API. +# +# null and bool explanations may be outdated. + +__END__ + +#line 1487 diff --git a/lib/RT/Extension/Import/CSV.pm b/lib/RT/Extension/Import/CSV.pm new file mode 100644 index 0000000..502d584 --- /dev/null +++ b/lib/RT/Extension/Import/CSV.pm @@ -0,0 +1,1552 @@ +use strict; +use warnings; +package RT::Extension::Import::CSV; + +use Text::CSV_XS; +use Test::MockTime 'restore_time'; + +our $VERSION = '0.01'; + +our( $CurrentRow, $CurrentLine, $UniqueFields ); + +sub _column { + ref($_[0]) ? (ref($_[0]) eq "CODE" ? + "code reference" : + "static value '${$_[0]}'") + : "column $_[0]" +} + +my %dispatch = ( + user => '_run_users', + ticket => '_run_tickets', + transaction => '_run_transactions', + article => '_run_articles', +); + +sub run { + my $class = shift; + my %args = ( + Type => undef, + @_, + ); + + my $type = delete $args{Type} // ''; + my $method = $dispatch{$type}; + if ( $method ) { + return $class->$method( %args ); + } + else { + $RT::Logger->error( "Invalid type: $type" ); + return ( 0, 0, 0 ); + } +} + +sub _run_users { + my $class = shift; + my %args = ( + CurrentUser => undef, + File => undef, + Update => undef, + Insert => undef, + @_, + ); + my $field2csv = $RT::Config->Get( 'UsersImportFieldMapping' ); + my $csv2fields = {}; + push @{ $csv2fields->{ $field2csv->{$_} } }, $_ for grep { not ref $field2csv->{$_} } keys %{$field2csv}; + + my ($header, @items) = $class->parse_csv( $args{File}, force => $args{Force} ); + unless (@items) { + $RT::Logger->warning( "No items found in file $args{File}" ); + return (0, 0, 0); + } + + $RT::Logger->debug( "Found unused column '$_'" ) + for grep {$_ ne 'U' && $_ ne '_line' && not $csv2fields->{$_}} keys %{ $items[0] }; + $RT::Logger->warning( "No column $_ found for @{$csv2fields->{$_}}" ) + for grep {not exists $items[0]->{$_} } keys %{ $csv2fields }; + + $RT::Logger->debug( 'Found ' . scalar(@items) . ' record(s)' ); + my ( $created, $updated, $skipped ) = ( 0 ) x 3; + my $row = 1; # Because of header row + for my $item ( @items ) { + local $CurrentRow = ++$row; + local $CurrentLine = $item->{_line}; + $RT::Logger->debug( "Start processing" ); + next unless grep { defined $_ && /\S/ } values %{ { %$item, _line => undef } }; + my $user = RT::User->new( $args{CurrentUser} ); + my $current_user = $args{CurrentUser}; + + # only insert for now, no update needed here yet. + my %args; + + for my $field (keys %$field2csv ) { + my $value = $class->get_value($field2csv->{$field}, $item); + next unless defined $value and length $value; + $value =~ s!;$!!; # email values contain extra ";" + $args{$field} = $value; + } + + $user->LoadByEmail( $args{EmailAddress} ); + if ( $user->id ) { + $RT::Logger->info( "Found existing user $args{EmailAddress}, skipping" ); + $skipped++; + } + else { + my $user = $class->load_or_create_user( CurrentUser => $current_user, %args ); + if ( $user ) { + $created++; + $RT::Logger->info( "Created user $args{EmailAddress}" ); + } + else { + $RT::Logger->error( "Failed to create user $args{EmailAddress}, skipping" ); + $skipped++; + } + } + } + return ( $created, $updated, $skipped ); +} + + +sub _run_transactions { + my $class = shift; + my %args = ( + CurrentUser => undef, + File => undef, + Update => undef, + Insert => undef, + @_, + ); + + my $field2csv = $RT::Config->Get('TransactionsImportFieldMapping'); + my $csv2fields = {}; + push @{$csv2fields->{ $field2csv->{$_} }}, $_ + for grep { not ref $field2csv->{$_} } keys %{$field2csv}; + + my ($header, @items) = $class->parse_csv( $args{File}, force => $args{Force} ); + unless ( @items ) { + $RT::Logger->warning( "No items found in file $args{File}" ); + return ( 0, 0, 0 ); + } + + $RT::Logger->warning( "No column $_ found for @{$csv2fields->{$_}}" ) + for grep {not exists $items[0]->{$_} } keys %{ $csv2fields }; + + $RT::Logger->debug( 'Found ' . scalar( @items ) . ' record(s)' ); + my ( $created, $updated, $skipped ) = ( 0 ) x 3; + my $row = 0; + for my $item ( @items ) { + local $CurrentRow = ++$row; + local $CurrentLine = $item->{_line}; + $RT::Logger->debug( "Start processing" ); + + next unless grep { defined $_ && /\S/ } values %{ { %$item, _line => undef } }; + + my $TicketId = $class->get_value($field2csv->{TicketID} , $item ); + + my $ticket = RT::Ticket->new( $args{CurrentUser} ); + $ticket->Load( $TicketId ); + if ( !$ticket->id ) { + $RT::Logger->error( "Failed to load ticket $TicketId, skipping" ); + $skipped++; + next; + } + my $mime = MIME::Entity->build( + Type => $item->{ContentType} || 'text/plain', + Charset => "UTF-8", + Data => [ Encode::encode( "UTF-8", $class->get_value($field2csv->{'Content'},$item) ) ], + ); + if($class->get_value($field2csv->{'Subject'},$item)) { + $mime->head->add( 'Subject' => Encode::encode( "UTF-8", $class->get_value($field2csv->{'Subject'},$item) ) ); + } + # Add any attachments + if ( $item->{$field2csv->{Attachment}} ) { + if ( -e $item->{$field2csv->{Attachment}} ) { + $mime->attach( + Path => $item->{$field2csv->{Attachment}}, + Type => $item->{$field2csv->{AttachmentContentType}} || 'application/octet-stream', + + ); + } + else { + $RT::Logger->error( "Could not load attachment: $item->{$field2csv->{Attachment}}" ); + } + } + + my $method = $class->get_value($field2csv->{'Type'}, $item); + + my ( $txn, $msg ); + if ( $method eq 'EmailRecord' ) { + my $msgid = Encode::decode( "UTF-8", $mime->head->get('Message-ID') ); + chomp $msgid; + + my $transaction = RT::Transaction->new( $ticket->CurrentUser ); + ( $txn, $msg ) = $transaction->Create( + Ticket => $ticket->Id, + Type => $method, + Data => $msgid, + MIMEObj => $mime, + ActivateScrips => 0 + ); + + if ( $txn ) { + $created++; + } + else { + $RT::Logger->warning( "Could not record outgoing message transaction: $msg" ); + } + } + else { + ( $txn, $msg ) = $ticket->$method( MIMEObj => $mime ); + if ( $txn ) { + $created++; + } + else { + $RT::Logger->error( "Failed to create transaction: $msg" ); + } + } + my $txn_object = RT::Transaction->new( RT->SystemUser ); + $txn_object->Load( $txn ); + + for my $fieldname (keys %{ $field2csv }) { + if ($fieldname =~ /^CF\.(.*)/) { + my $value = $class->get_value( $field2csv->{$fieldname}, $item ); + my $cfname = $1; + + my $cf = RT::CustomField->new( $args{CurrentUser} ); + $cf->LoadByName( + Name => $cfname, + LookupType => RT::Transaction->CustomFieldLookupType, + ObjectId => $ticket->Queue, + IncludeGlobal => 1, + ); + if ( $cf->Id ) { + if ($cf->Type eq "DateTime") { + my $args = { Content => $value }; + $value = $args->{Content}; + } elsif ($cf->Type eq "Date") { + my $args = { Content => $value }; + $cf->_CanonicalizeValueDate( $args ); + $value = $args->{Content}; + } + + my @current = @{$txn_object->CustomFieldValues( $cf->id )->ItemsArrayRef}; + next if grep {$_->Content and $_->Content eq $value} @current; + + my ($ok, $msg) = $txn_object->AddCustomFieldValue( + Field => $cf->id, + Value => $value, + ); + unless ($ok) { + $RT::Logger->error("Failed to set CF $cfname to $value: $msg"); + } + } + else { + $RT::Logger->warning( + "Missing custom field $cfname for "._column($field2csv->{$fieldname}).", skipping"); + next; + } + } + # For now hard code the created column + elsif ($fieldname =~ /^(Created)$/) { + my $date = RT::Date->new( RT->SystemUser ); + my $value = $class->get_value( $field2csv->{'Created'}, $item ); + $date->Set( Format => 'iso', Value => $value ); + + ( my $ok, $msg ) = $txn_object->__Set( Field => 'Created', Value => $date->ISO ); + $RT::Logger->error( "Failed to set Created on transaction: $msg" ) unless $ok; + } + } + } + return ( $created, $updated, $skipped ); +} + +my %ticket_extra_fields = ( + map { $_ => 1 } qw/T status/ +); + +sub _run_tickets { + my $class = shift; + my %args = ( + CurrentUser => undef, + File => undef, + Update => undef, + Insert => undef, + InsertUpdate => undef, + @_, + ); + + my $field2csv = $RT::Config->Get('TicketsImportFieldMapping'); + my $force = $args{Force}; + my $csv2fields = {}; + push @{$csv2fields->{ $field2csv->{$_} }}, $_ + for grep { not ref $field2csv->{$_} } keys %{$field2csv}; + + # Right now, the CSV configuration *requires* setting 'Queue' to a + # static string reference. That means each CSV file can only + # contain tickets for a single Queue. + # + # In the future, we may want to extend Queue column handling so + # that CSVs can contain tickets for different queues. That will + # require testing each CF and CR that the row has values for to + # make sure they are applied to the given Queue. + unless (ref($field2csv->{'Queue'}) eq "SCALAR") { + $RT::Logger->error( "Default Queue is not defined. Make sure Queue value is a reference to a string." ); + return (0, 0, 0); + } + + my $default_queue = RT::Queue->new( $args{CurrentUser} ); + $default_queue->Load(${$field2csv->{Queue}}); + unless ( $default_queue->Id ) { + RT->Logger->error( "Could not load queue: " . $field2csv->{Queue} ); + } + + if (scalar RT->Config->Get('TicketsImportUniqueCFs') && RT->Config->Get('TicketsImportTicketIdField') ) { + RT->Logger->error( "Provided 'TicketsImportUniqueCFs' and 'TicketsImportTicketIdField' config values, can only have one." ); + return (0, 0, 0); + } + + my @unique = (); + # If we are updating based on existing ticket ID's then we shouldn't need to look at custom fields + if ( !RT->Config->Get('TicketsImportTicketIdField') ) { + @unique = RT->Config->Get('TicketsImportUniqueCFs') if RT->Config->Get('TicketsImportUniqueCFs'); + } + + if ( !scalar @unique && !RT->Config->Get('TicketsImportTicketIdField')) { + if ( $args{Update} or $args{InsertUpdate} ) { + $RT::Logger->error( "TicketsImportUniqueCFs or TicketsImportTicketIdField is not set and is required for updating tickets" ); + return ( 0, 0, 0 ); + } + + if ( !$args{Insert} ) { + $RT::Logger->error( "TicketsImportUniqueCFs or TicketsImportTicketIdField is not set. Use --insert to create tickets" ); + return ( 0, 0, 0 ); + } + } + + # Confirm we can load the configured unique CFs and save the ids for later + my %unique_cf_objs; + if ( scalar @unique) { + foreach my $unique ( @unique ){ + my $unique_cf = RT::CustomField->new( $args{CurrentUser} ); + $unique_cf->LoadByName( + Name => $unique, + LookupType => RT::Ticket->CustomFieldLookupType, + ObjectId => $default_queue->id, + IncludeGlobal => 1, + ); + unless ($unique_cf->id) { + $RT::Logger->error( "Can't find custom field $unique for RT::Tickets" ); + return (0, 0, 0); + } + $unique_cf_objs{"$unique"} = $unique_cf; + } + } + + my %cfmap; + my %crmap; + for my $fieldname (keys %{ $field2csv }) { + if ($fieldname =~ /^CF\.(.*)/) { + my $cfname = $1; + my $cf = RT::CustomField->new( $args{CurrentUser} ); + $cf->LoadByName( + Name => $cfname, + LookupType => RT::Ticket->CustomFieldLookupType, + ObjectId => $default_queue->id, + IncludeGlobal => 1, + ); + if ( $cf->id ) { + $cfmap{$cfname} = $cf; + } else { + $RT::Logger->warning( + "Missing custom field $cfname for "._column($field2csv->{$fieldname}).", skipping"); + delete $field2csv->{$fieldname}; + } + } elsif ($fieldname =~ /^CR\.(.*)/) { + # no-op for now + my $crname = $1; + my $cr = RT::CustomRole->new( $args{CurrentUser} ); + $cr->Load( $crname ); + if ( $cr->id ) { + $crmap{$crname} = $cr; + } + else { + $RT::Logger->warning( + "Missing custom role $crname for " . _column( $field2csv->{$fieldname} ) . ", skipping" ); + delete $field2csv->{$fieldname}; + } + } elsif ($fieldname =~ /^(id|Creator|LastUpdated|Created|Queue|Requestor|Cc|AdminCc|SquelchMailTo|Type|Owner| + Subject|Priority|InitialPriority|FinalPriority|Status|TimeEstimated|TimeWorked|TimeLeft|Starts|Due|MIMEObj| + Comment|Correspond|MemberOf|Parents|Parent|Members|Member|Children|Child|HasMember|RefersTo|ReferredToBy| + DependsOn|DependedOnBy)$/x) { + # no-op, these are fine + } else { + $RT::Logger->warning( + "Unknown ticket field $fieldname for "._column($field2csv->{$fieldname}).", skipping"); + delete $field2csv->{$fieldname}; + } + } + + my %tolerant_roles = map { $_ => 1 } @{ RT->Config->Get('TicketTolerantRoles') || [] }; + + my ($header, @items) = $class->parse_csv( $args{File}, force => $args{Force} ); + unless (@items) { + $RT::Logger->warning( "No items found in file $args{File}" ); + return (0, 0, 0); + } + + $RT::Logger->debug( "Found unused column '$_'" ) + for grep { $_ ne '_line' && !$ticket_extra_fields{$_} && !$csv2fields->{$_}} keys %{ $items[0] }; + $RT::Logger->warning( "No column $_ found for @{$csv2fields->{$_}}" ) + for grep {not exists $items[0]->{$_} } keys %{ $csv2fields }; + + $RT::Logger->debug( 'Found ' . scalar(@items) . ' record(s)' ); + my ( $created, $updated, $skipped ) = (0) x 3; + my @skipped; # Save skipped records for output to errors file + + my $row = 1; # Because of header row + ROW: + for my $item (@items) { + local $CurrentRow = ++$row; + local $CurrentLine = $item->{_line}; + $RT::Logger->debug( "Start processing" ); + next unless grep { defined $_ && /\S/ } values %{ { %$item, _line => undef } }; + + my $tickets = RT::Tickets->new( $args{CurrentUser} ); + + # Exclude statuses configured within ExcludeStatusesOnSearch from the loaded tickets + my @excluded_statuses; + @excluded_statuses = RT->Config->Get('ExcludeStatusesOnSearch') + if RT->Config->Get('ExcludeStatusesOnSearch'); + + if ( scalar @excluded_statuses ) { + foreach my $status ( @excluded_statuses ) { + unless ( $default_queue->LifecycleObj->IsValid( lc($status) ) ) { + $RT::Logger->warning( "Status '$status' is not valid. Tickets match will not exclude '$status'" ); + next; + } + + $tickets->Limit( + FIELD => 'Status', + VALUE => lc($status), + OPERATOR => '!=', + ENTRYAGGREGATOR => 'AND', + ); + } + } + + my $unique_fields = 'Unique field data: '; + + if ( scalar @unique ) { + my $id_value; + + my @unique_fields_data; + + foreach my $unique ( @unique ){ + $id_value = $class->get_value( $field2csv->{"CF.$unique"}, $item ) // ''; + push @unique_fields_data, "$unique: $id_value"; + if ( length $id_value ) { + $tickets->_LimitCustomField( + CUSTOMFIELD => $unique_cf_objs{"$unique"}, + VALUE => $id_value, + ); + } + else{ + $tickets->_LimitCustomField( + CUSTOMFIELD => $unique_cf_objs{"$unique"}, + VALUE => undef, + OPERATOR => 'IS', + ); + } + } + + $unique_fields .= join( ', ', @unique_fields_data ); + } + + # set within this scope for RT::Logger->add_callback + local $UniqueFields = $unique_fields; + + if ( RT->Config->Get('TicketsImportTicketIdField') && $args{'Update'} ) { + my $value = $class->get_value( RT->Config->Get('TicketsImportTicketIdField'), $item ); + unless ( $value ) { + RT->Logger->error( "Invalid \$TicketsImportTicketIdField: '".RT->Config->Get('TicketsImportTicketIdField')."' value provided, unable to find field mapping" ); + $skipped++; + push @skipped, $item; + next; + } + + $tickets->Limit( FIELD => 'Id', VALUE => $value ); + } + + if ( $tickets->Count ) { + my $ticket; + + if ( $tickets->Count > 1 ) { + if ( RT->Config->Get('TicketsImportTicketIdField') ) { + my $id = $class->get_value( RT->Config->Get('TicketsImportTicketIdField'), $item ); + $RT::Logger->warning( "Found multiple tickets IDs, for $id, skipping." ); + } + else { + $RT::Logger->warning( "Found multiple tickets for CFs, skipping. $unique_fields" ); + } + $skipped++; + push @skipped, $item; + next; + } + else { + $ticket = $tickets->First; + my $ticket_id = $ticket->Id; + if ( RT->Config->Get('TicketsImportTicketIdField') ) { + $RT::Logger->debug( "Found existing ticket($ticket_id)" ); + } + else { + $RT::Logger->debug( "Found existing ticket($ticket_id) for CFs. $unique_fields" ); + } + } + + unless ( $args{Update} or $args{InsertUpdate} ) { + if ( RT->Config->Get('TicketsImportTicketIdField') ) { + my $id = $class->get_value( RT->Config->Get('TicketsImportTicketIdField'), $item ); + $RT::Logger->debug( + "Found existing ticket but no '--update' or '--insert-update' option, skipping. $id" + ); + } + else { + $RT::Logger->debug( + "Found existing ticket but no '--update' or '--insert-update' option, skipping. $unique_fields" + ); + } + $skipped++; + push @skipped, $item; + next; + } + + + if ( my $callback = RT->Config->Get('PreTicketChangeCallback') ) { + my ( $ret, $msg ) = $callback->( + TicketObj => $ticket, + Row => $item, + Type => 'Update', + CurrentUser => $args{CurrentUser}, + ); + if ( !$ret ) { + $RT::Logger->debug( "PreTicketChangeCallback returned false, skipping. " . ( $msg // '' ) ); + $skipped++; + push @skipped, $item; + next; + } + } + + my $changes; + my $invalid; + for my $field ( keys %$field2csv ) { + my $value = $class->get_value( $field2csv->{$field}, $item ); + unless ( defined $value and length $value ) { + if ( grep { $field eq $_ } @{ RT->Config->Get( 'TicketMandatoryFields' ) || [] } ) { + $RT::Logger->error( "Missing mandatory $field, skipping. $unique_fields" ); + $invalid = 1; + } + else { + next; + } + } + + if ($field =~ /^CF\.(.*)/) { + my $cfname = $1; + + if ($cfmap{$cfname}->Type eq "DateTime") { + my $args = { Content => $value }; + # $cfmap{$cfname}->_CanonicalizeValueDateTime( $args ); + $value = $args->{Content}; + } elsif ($cfmap{$cfname}->Type eq "Date") { + my $args = { Content => $value }; + $cfmap{$cfname}->_CanonicalizeValueDate( $args ); + $value = $args->{Content}; + } + + my @current = @{$ticket->CustomFieldValues( $cfmap{$cfname}->id )->ItemsArrayRef}; + next if grep {$_->Content and $_->Content eq $value} @current; + + $changes++; + my ($ok, $msg) = $ticket->AddCustomFieldValue( + Field => $cfmap{$cfname}->id, + Value => $value, + ); + unless ($ok) { + $RT::Logger->error("Failed to set CF $cfname to $value: $msg"); + } + } elsif ($field =~ /^CR\.(.*)/) { + my $crname = $1; + # we only want to check members that are directly added to the group + my %members = map { $_->id => $_ } + @{ $ticket->RoleGroup( $crmap{$crname}->GroupType )->UserMembersObj( Recursively => 0 )->ItemsArrayRef }; + + my @values = $class->parse_email_address( $value ); + for my $value ( @values ) { + my $user = $class->load_or_create_user( CurrentUser => $args{CurrentUser}, EmailAddress => $value->address ); + if ( $user ) { + if ( $members{$user->id} ) { + delete $members{$user->id}; + } + else { + my ( $ok, $msg ) + = $ticket->AddRoleMember( PrincipalId => $user->PrincipalId, Type => $crmap{$crname}->GroupType ); + if ( $ok ) { + $changes++; + } + else { + $RT::Logger->error( "Failed to add $value to $field: $msg" ); + } + } + } + else { + $RT::Logger->error( "Failed to find user with email '$value'" ); + } + } + # delete old ones + for my $id ( keys %members ) { + next unless $ticket->RoleGroup( $crmap{$crname}->GroupType )->HasMember( $id ); + my ( $ok, $msg ) = $ticket->DeleteRoleMember( PrincipalId => $id, Type => $crmap{$crname}->GroupType ); + if ( $ok ) { + $changes++; + } + else { + $RT::Logger->error( "Failed to delete " . $members{$id}->Name . " from $field: $msg" ); + } + } + } elsif ($field =~ /^(?:Requestor|Cc|AdminCc)$/) { + my %members = map { $_->id => $_ } + @{ $ticket->RoleGroup( $field )->UserMembersObj( Recursively => 0 )->ItemsArrayRef }; + + my @values = $class->parse_email_address( $value ); + for my $value ( @values ) { + my $user = $class->load_or_create_user( CurrentUser => $args{CurrentUser}, EmailAddress => $value->address ); + if ( $user ) { + if ( $members{$user->id} ) { + delete $members{$user->id}; + } + else { + my ( $ok, $msg ) + = $ticket->AddRoleMember( PrincipalId => $user->PrincipalId, Type => $field ); + if ( $ok ) { + $changes++; + } + else { + $RT::Logger->error( "Failed to add $value to $field: $msg" ); + } + } + } + else { + $RT::Logger->error( "Failed to find user with email '$value'" ); + } + } + # delete old ones + for my $id ( keys %members ) { + next unless $ticket->RoleGroup( $field )->HasMember( $id ); + my ( $ok, $msg ) = $ticket->DeleteRoleMember( PrincipalId => $id, Type => $field ); + if ( $ok ) { + $changes++; + } + else { + $RT::Logger->error( "Failed to delete " . $members{$id}->Name . " from $field: $msg" ); + } + } + } elsif ( $ticket->_CoreAccessible->{$field}{write} ) { + if ($field eq "Queue") { + my $queue = RT::Queue->new( $args{CurrentUser} ); + $queue->Load( $value ); + $value = $queue->id; + } + + if ( $field eq 'Owner' ) { + $value =~ s!;$!!; + my $user = $class->load_or_create_user( CurrentUser => $args{CurrentUser}, EmailAddress => $value ); + if ( $user ) { + $value = $user->id; + } + else { + $RT::Logger->error( "Failed to find user with email '$value'" ); + } + } + + if ($ticket->$field ne $value) { + my $method = "Set" . $field; + my ($ok, $msg) = $ticket->$method( $value ); + if ( $ok ) { + $changes++; + } + else { + $RT::Logger->error( "Failed to set $field to $value: $msg" ); + } + } + } elsif ($field =~ /^(?:Correspond|Comment)$/) { + my ($ok, $msg) = $ticket->$field( Content => $value ); + if ( $ok ) { + $changes++; + } + else { + $RT::Logger->error( "Failed to $field on ticket with content $value: $msg" ); + } + } + } + + if ($invalid) { + $skipped++; + push @skipped, $item; + next; + } + + if ($changes) { + $RT::Logger->debug( "Ticket " . $ticket->id . " updated. $unique_fields" ); + $updated++; + if ( my $callback = RT->Config->Get('PostTicketChangeCallback') ) { + $callback->( TicketObj => $ticket, Row => $item, + Type => 'Update', CurrentUser => $args{CurrentUser} ); + } + } else { + $RT::Logger->debug( "Ticket " . $ticket->id . " skipped. No updates required." ); + $skipped++; + push @skipped, $item; + } + } + else { + # No existing tickets found, consider insert + unless ( $args{Insert} or $args{InsertUpdate} ) { + $RT::Logger->debug( + "No existing tickets found and no '--insert' or '--insert-update' option, skipping. $unique_fields" + ); + $skipped++; + push @skipped, $item; + next; + } + + if ( my $callback = RT->Config->Get('PreTicketChangeCallback') ) { + my ( $ret, $msg ) = $callback->( + Row => $item, + Type => 'Create', + CurrentUser => $args{CurrentUser}, + ); + if ( !$ret ) { + $RT::Logger->debug( "PreTicketChangeCallback returned false, skipping. " . ( $msg // '' ) ); + $skipped++; + push @skipped, $item; + next; + } + } + + my $ticket = RT::Ticket->new( $args{CurrentUser} ); + my $current_user = $args{CurrentUser}; + my %args; + + for my $field (keys %$field2csv ) { + my $value = $class->get_value($field2csv->{$field}, $item); + unless ( defined $value and length $value ) { + if ( grep { $field eq $_ } @{ RT->Config->Get( 'TicketMandatoryFields' ) || [] } ) { + $RT::Logger->error( "Missing mandatory $field, skipping. $unique_fields" ); + $invalid = 1; + } + else { + next; + } + } + + if ($field =~ /^CF\.(.*)/) { + my $cfname = $1; + my $args = { Content => $value }; + my ( $ret, $msg ) = $cfmap{$cfname}->_CanonicalizeValue( $args ); + + # Date cfs return 1970-01-01 if it can't extrat dates + if ( $cfmap{$cfname}->Type =~ /^Date(?:Time)?$/ && $args->{Content} =~ /^1970-01-01/ ) { + $ret = 0; + } + + # Verify select-one type CF values are one of the allowed values for that CF + if ( $cfmap{$cfname}->Type eq 'Select' ) { + my @allowed_values = @{ $cfmap{$cfname}->Values->ItemsArrayRef }; + $ret = 0 unless grep { $_->Name eq $value } @allowed_values; + } + + if ($ret) { + $args{ "CustomField-" . $cfmap{$cfname}->id } = $value; + } + elsif ($force) { + RT->Logger->error("Invalid CF $cfname value '$value', creating without it"); + } + else { + RT->Logger->error( "Invalid CF $cfname value '$value', skipping. $unique_fields" ); + $invalid = 1; + } + } elsif ($field =~ /^CR\.(.*)/) { + my $crname = $1; + my @values = $class->parse_email_address( $value ); + + if ( !@values ) { + if ( $force || $tolerant_roles{$field} ) { + RT->Logger->error("Failed to extract email from '$value', creating without it"); + } + else { + RT->Logger->error("Failed to extract email from '$value', skipping. $unique_fields"); + $invalid = 1; + } + } + + my @emails; + for my $value ( @values ) { + my $user = $class->load_or_create_user( CurrentUser => $current_user, EmailAddress => $value->address ); + if ( $user ) { + push @emails, $value; + } + elsif ( $force || $tolerant_roles{$field} ) { + RT->Logger->error( + "Failed to find user with email '$value', creating without it" ); + } + else { + RT->Logger->error( "Failed to find user with email '$value', skipping. $unique_fields" ); + $invalid = 1; + } + } + $args{ $crmap{$crname}->GroupType } = join ', ', @emails; + } elsif ($field =~ /^(?:Requestor|Cc|AdminCc)$/) { + my @values = $class->parse_email_address( $value ); + + if ( !@values ) { + if ( $force || $tolerant_roles{$field} ) { + RT->Logger->error("Failed to extract email from '$value', creating without it"); + } + else { + RT->Logger->error("Failed to extract email from '$value', skipping. $unique_fields"); + $invalid = 1; + } + } + + my @emails; + for my $value ( @values ) { + my $user = $class->load_or_create_user( CurrentUser => $current_user, EmailAddress => $value->address ); + if ( $user ) { + push @emails, $value; + } + elsif ( $force || $tolerant_roles{$field} ) { + RT->Logger->error( + "Failed to find user with email '$value', creating without it" ); + } + else { + RT->Logger->error( "Failed to find user with email '$value', skipping. $unique_fields" ); + $invalid = 1; + } + } + $args{ $field } = join ', ', @emails; + } elsif ($field eq 'Owner' && $value) { + $value =~ s!;$!!; + my $user = $class->load_or_create_user( CurrentUser => $current_user, EmailAddress => $value ); + if ( $user && $user->HasRight( Right => 'OwnTicket', Object => $default_queue ) ) { + $args{$field} = $user->id; + } + elsif ( $force || $tolerant_roles{$field} ) { + if ( $user ) { + RT->Logger->error( + "User with email '$value' doesn't have OwnTicket right, creating with owner as Nobody" + ); + } + else { + RT->Logger->error( "Failed to find owner with email '$value', creating with owner as Nobody" ); + } + delete $args{$field}; + } + else { + if ( $user ) { + RT->Logger->error( + "User with email '$value' doesn't have OwnTicket right, skipping. $unique_fields" + ); + } + else { + RT->Logger->error( + "Failed to find owner with email '$value', skipping. $unique_fields" + ); + } + $invalid = 1; + } + } else { + $args{$field} = $value; + } + + if ( $field =~ /^(?:Correspond|Comment)$/ ) { + $args{'MIMEObj'} = MIME::Entity->build( + Type => "text/plain", + Charset => "UTF-8", + Data => Encode::encode("UTF-8", $value), + ); + } + } + + my $status = delete( $args{Status} ); + if ( $status && !$default_queue->LifecycleObj->IsValid($status) ) { + if ($force) { + RT->Logger->error("Status '$status' is not valid, creating without it"); + } + else { + RT->Logger->error("Status '$status' is not valid, skipping. $unique_fields"); + $invalid = 1; + } + } + + my $created_date = delete $args{Created}; + if ( $created_date ) { + my $date = RT::Date->new( RT->SystemUser ); + $date->Set( Format => 'iso', Value => $created_date ); + if ( !$date->Unix ) { + if ($force) { + RT->Logger->error("Created date '$created_date' is not valid, creating without it"); + } + else { + RT->Logger->error("Created date '$created_date' is not valid, skipping. $unique_fields"); + $invalid = 1; + } + } + } + + if ($invalid) { + $skipped++; + push @skipped, $item; + next; + } + + my ($ok, $txnobj, $msg) = $ticket->Create( %args ); + + if ($ok) { + $created++; + } else { + $RT::Logger->error("Failed to create ticket: $msg, skipping. $unique_fields"); + $skipped++; + push @skipped, $item; + next ROW; + } + + if ($status && $status ne $ticket->Status) { + ($ok, $msg) = $ticket->__Set(Field => 'Status', Value => $status); + $RT::Logger->error("Failed to set Status on ticket: $msg") unless $ok + } + + if ( $created_date ) { + my $date = RT::Date->new( RT->SystemUser ); + $date->Set( Format => 'iso', Value => $created_date ); + ( $ok, $msg ) = $ticket->__Set( Field => 'Created', Value => $date->ISO ); + $RT::Logger->error("Failed to set Created on ticket: $msg") unless $ok; + + my $txns = $ticket->Transactions; + $txns->Limit( FIELD => 'Type', VALUE => 'Create' ); + if ( my $txn = $txns->First ) { + ( $ok, $msg ) = $txn->__Set( Field => 'Created', Value => $date->ISO ); + $RT::Logger->error( "Failed to set Created on ticket create transaction: $msg" ) + unless $ok; + } + } + + if ( my $callback = RT->Config->Get('PostTicketChangeCallback') ) { + $callback->( TicketObj => $ticket, Row => $item, + Type => 'Create', CurrentUser => $args{CurrentUser} ); + } + } + } + + # Convert skipped hashrefs into correctly ordered arrays based on the header + my @skipped_refs; + foreach my $item_ref ( @skipped ){ + my @skipped_line; + foreach my $column ( @$header ){ + next if $column eq '_line'; + push @skipped_line, $item_ref->{$column}; + } + push @skipped_refs, \@skipped_line; + } + + # Prepend the header line if we found any skipped items + unshift @skipped_refs, $header if scalar @skipped; + + return ( $created, $updated, $skipped, \@skipped_refs ); +} + +sub get_value { + my $class = shift; + my ($from, $data) = @_; + if (not ref $from) { + return $data->{$from}; + } elsif (ref($from) eq "CODE") { + return $from->($data); + } else { + return $$from; + } +} + +sub parse_csv { + my $class = shift; + my $file = shift; + my %args = @_; + + my @rows; + + open my $fh, '<', $file or die "failed to read $file: $!"; + while (<$fh>) { + if ( /\r\r\n/ ) { + RT->Logger->error( "Line $. contains invalid characters" . '(\r\r\n), skipping' ); + return; + } + } + + my $csv = Text::CSV_XS->new( + { + sep_char => ',', + binary => 1, + %{ RT->Config->Get('CSVOptions') || {} }, + } + ); + + close $fh; + open $fh, '<', $file or die "failed to read $file: $!"; + my $header = $args{header} || $csv->getline($fh); + my @items; + + unless ( $header ){ + RT->Logger->error("Error reading header line from file $file, stopping import"); + return $header, @items; + } + + my $previous_line = $. || 0; + while ( my $row = $csv->getline($fh) ) { + my $item = { _line => $previous_line + 1 }; + # get around the extra and suspicious column + @$row = grep { !/Ticket was imported from TTP/ } @$row if @$header <= @$row; + for ( my $i = 0 ; $i < @$header ; $i++ ) { + if ( $header->[$i] ) { + $item->{ $header->[$i] } = $row->[$i]; + } + } + + push @items, $item; + $previous_line = $.; + } + + if ( !$csv->eof ) { + RT->Logger->error( $csv->error_diag() ); + exit 1 unless $args{force}; + } + close $fh; + return $header, @items; +} + +sub set_fixed_time { + my $class = shift; + my $value = shift; + my $date = RT::Date->new( RT->SystemUser ); + $date->Set( Format => 'iso', Value => $value ); + if ( $date->Unix > 0 ) { + Test::MockTime::set_fixed_time( $date->Unix ); + } + else { + $RT::Logger->warning( "Invalid datetime: $value" ); + } +} + +sub load_or_create_user { + my $class = shift; + my %args = @_; + my $user = RT::User->new( delete $args{CurrentUser} ); + $user->LoadByEmail( $args{EmailAddress} ); + return $user if $user->id; + + my ( $ok, $msg ) = $user->Create( Privileged => 1, Name => $args{EmailAddress}, %args ); + if ($ok) { + return $user; + } + + my $arg = { EmailAddress => $args{EmailAddress} }; + if ( $msg eq 'Name in use' && $user->CanonicalizeUserInfoFromExternalAuth($arg) ) { + if ( $arg->{Name} ) { + $user = RT::User->new( $user->CurrentUser ); + $user->Load( $arg->{Name} ); + if ( $user->id ) { + RT->Logger->warning( + "Found user with same Name($arg->{Name}) but provided email address " + . $args{EmailAddress} . " differs from RT email address: " . $user->EmailAddress ); + my ( $ret, $msg ) = $user->SetEmailAddress( $args{EmailAddress} ); + if ($ret) { + RT->Logger->info( "Updated user #" . $user->Id . " EmailAddress to $args{EmailAddress}" ); + } + else { + RT->Logger->warning( + "Couldn't update user #" . $user->Id . " EmailAddress to $args{EmailAddress}: $msg" ); + } + return $user; + } + } + } + return undef; +} + +sub _run_articles { + my $class = shift; + my %args = ( + CurrentUser => undef, + File => undef, + Update => undef, + Insert => undef, + @_, + ); + + my $article_class_name = $args{ArticleClass}; + my $article_class = RT::Class->new( RT->SystemUser ); + my ( $ret, $msg ) = $article_class->Load( $article_class_name ); + if ( !$ret ) { + $RT::Logger->error("Failed to load article class $article_class_name: $msg"); + return ( 0, 0, 0 ); + } + + my $field2csv = $RT::Config->Get('ArticlesImportFieldMapping'); + my $csv2fields = {}; + push @{ $csv2fields->{ $field2csv->{$_} } }, $_ for grep { not ref $field2csv->{$_} } keys %{$field2csv}; + + my ($header, @items) = $class->parse_csv( $args{File}, force => $args{Force} ); + unless (@items) { + $RT::Logger->warning("No items found in file $args{File}"); + return ( 0, 0, 0 ); + } + + $RT::Logger->debug("Found unused column '$_'") + for grep { $_ ne '_line' && not $csv2fields->{$_} } keys %{ $items[0] }; + $RT::Logger->warning("No column $_ found for @{$csv2fields->{$_}}") + for grep { not exists $items[0]->{$_} } keys %{$csv2fields}; + + $RT::Logger->debug( 'Found ' . scalar(@items) . ' record(s)' ); + my ( $created, $updated, $skipped ) = (0) x 3; + my %cf_id; + my $update = $args{Update}; + + my $row = 1; # Because of header row + for my $item (@items) { + local $CurrentRow = ++$row; + local $CurrentLine = $item->{_line}; + $RT::Logger->debug("Start processing"); + next unless grep { defined $_ && /\S/ } values %{ { %$item, _line => undef } }; + my $article = RT::Article->new( $args{CurrentUser} ); + my $current_user = $args{CurrentUser}; + + # only insert for now, no update needed here yet. + my %args; + + for my $field ( keys %$field2csv ) { + my $value = $class->get_value( $field2csv->{$field}, $item ); + next unless defined $value and length $value; + if ( $field =~ /^CF\.(.+)/ ) { + my $name = $1; + if ( !$cf_id{$name} ) { + my $cf = RT::CustomField->new( RT->SystemUser ); + $cf->LoadByName( + Name => $name, + LookupType => RT::Article->CustomFieldLookupType, + ObjectId => $article_class->id, + IncludeGlobal => 1, + ); + if ($ret) { + $cf_id{$name} = $cf->id; + } + else { + $RT::Logger->error("Failed to load article custom field $name: $msg"); + } + } + $args{"CustomField-$cf_id{$name}"} = $value; + } + else { + $args{$field} = $value; + } + } + + $article->LoadByCols( Name => $args{Name} ); + if ( $article->id ) { + $RT::Logger->info("Found existing article $args{Name}"); + unless ($update) { + $RT::Logger->debug("Found existing article but without 'Update' option, skipping."); + $skipped++; + next; + } + + my $changed; + for my $field ( keys %args ) { + if ( $field =~ /CustomField-(\d+)/ ) { + my $cf_id = $1; + if ( $article->FirstCustomFieldValue($1) ne $args{$field} ) { + my ( $ret, $msg ) = $article->AddCustomFieldValue( + Field => $cf_id, + Value => $args{$field}, + ); + if ($ret) { + $changed ||= 1; + } + else { + $RT::Logger->error("Failed to set $field to $args{$field}: $msg"); + } + } + } + elsif ( $article->$field ne $args{$field} ) { + my $method = "Set$field"; + my ( $ret, $msg ) = $article->$method( $args{$field} ); + if ($ret) { + $changed ||= 1; + } + else { + $RT::Logger->error("Failed to set $field to $args{$field}: $msg"); + } + } + } + + if ($changed) { + $RT::Logger->debug("Updated article $args{Name} in class $article_class_name"); + $updated++; + } + else { + $RT::Logger->debug("Skipped article $args{Name} in class $article_class_name"); + $skipped++; + } + } + else { + my $article = RT::Article->new($current_user); + my ( $ret, $msg ) = $article->Create( Class => $article_class->id, %args ); + if ($ret) { + $created++; + $RT::Logger->info("Created article $args{Name} in class $article_class_name"); + } + else { + $RT::Logger->error("Failed to created article $args{Name} in class $article_class_name: $msg"); + $skipped++; + } + } + } + return ( $created, $updated, $skipped ); +} + + +# Based on RT::EmailParser::ParseEmailAddress, and also checks external auth +# if the value is a name and RT doesn't have corresponding user + +sub parse_email_address { + my $class = shift; + my $address_string = shift; + + $address_string =~ s/;/,/g; + + # Some broken mailers send: ""Vincent, Jesse"" . Hate + $address_string =~ s/\"\"(.*?)\"\"/\"$1\"/g; + + my @list = Email::Address::List->parse( + $address_string, + skip_comments => 1, + skip_groups => 1, + ); + my $logger = sub { + RT->Logger->error( "Unable to parse an email address from $address_string: " . shift ); + }; + + my @addresses; + foreach my $e (@list) { + if ( $e->{'type'} eq 'mailbox' ) { + if ( $e->{'not_ascii'} ) { + $logger->( $e->{'value'} . " contains not ASCII values" ); + next; + } + push @addresses, $e->{'value'}; + } + elsif ( $e->{'value'} =~ /^\s*([\w ]+)\s*$/ ) { + my $name = $1; + my $user = RT::User->new( RT->SystemUser ); + $user->Load($name); + if ( $user->id ) { + push @addresses, Email::Address->new( $user->Name, $user->EmailAddress ); + } + else { + my $args = { Name => $name }; + if ( $user->CanonicalizeUserInfoFromExternalAuth($args) ) { + push @addresses, Email::Address->new( $args->{EmailAddress} ); + } + else { + $logger->( $e->{'value'} . " is not a valid email address and is not user name" ); + } + } + } + else { + $logger->( $e->{'value'} . " is not a valid email address" ); + } + } + + RT::EmailParser->CleanupAddresses(@addresses); + + return @addresses; +} + +=head1 NAME + +RT-Extension-Import-CSV + +=head1 DESCRIPTION + +Import data into RT from CSVs. + +=head1 RT VERSION + +Works with RT 5. + +=head1 INSTALLATION + +=over + +=item C + +=item C + +=item C + +May need root permissions + +=item Edit your F + +Add this line: + + Plugin('RT::Extension::Import::CSV'); + +=item Clear your mason cache + + rm -rf /opt/rt5/var/mason_data/obj + +=item Restart your webserver + +=back + +=head1 CONFIGURATION + +The following configuration would be used to import a three-column CSV +of tickets, where the custom field C must be unique. +That option can accept multiple values and the combination of values +must find no existing tickets for insert, or a single ticket for update. +If multiple tickets match, the CSV row is skipped. + + Set( @TicketsImportUniqueCFs, ('Original Ticket ID') ); + Set( %TicketsImportFieldMapping, + 'Created' => 'Ticket-Create-Date', + 'CF.Original Ticket ID' => 'TicketID', + 'Subject' => 'name', + ); + +=head2 Excluding Existing Tickets By Status + +Some tickets will be opened, issues will be fixed, and the ticket will be marked +as closed. Later, the same asset (e.g., a server) may have a new ticket +opened for a newly found issue. In these cases, a new ticket should be +created and the previous ticket should not be re-opened. To instruct +the importer to exclude tickets in some statuses, set the following option: + + Set( @ExcludeStatusesOnSearch, ('fixed')); + +=head2 Constant values + +If you want to set an RT column or custom field to a static value for +all imported tickets, precede the "CSV field name" (right hand side of +the mapping) with a slash, like so: + + Set( %TicketsImportFieldMapping, + 'Queue' => \'General', + 'Created' => 'Ticket-Create-Date', + 'CF.Original TicketID' => 'TicketID', + 'Subject' => 'name', + ); + +Every imported ticket will now be added to the 'General' queue. This +feature is particularly useful for setting the queue, but may also be +useful when importing tickets from CSV sources you don't control (and +don't want to modify each time). + +=head2 Computed values + +You may also compute values during import, by passing a subroutine +reference as the value in the C<%TicketsImportFieldMapping>. This +subroutine will be called with a hash reference of the parsed CSV +row. In the following example, the subroutine assigned to the 'Status' +field takes the value in the 'status' CSV column and replaces +underscores with spaces. + + Set( %TicketsImportFieldMapping, + 'Queue' => \'General', + 'Created' => 'Ticket-Create-Date', + 'CF.Original TicketID' => 'TicketID', + 'Subject' => 'name', + 'Status' => sub { $_[0]->{status} =~ s/_/ /gr; }, + ); + +Using computed columns may cause false-positive "unused column" +warnings; these can be ignored. + +=head2 Mandatory fields + +To mark some ticket fields mandatory: + + Set( @TicketMandatoryFields, 'CF.Severity' ); + +Then rows without "CF.Severity" values will be skipped. + +=head2 Extra Options for Text::CSV_XS + +The CSV importer is configured to read the CSV import format determined when initially +testing. However, the Text::CSV_XS module is configurable and can handle different +CSV variations. You can pass through custom options using the configuration below. +Available options are described in the documentation for L. + + Set( %CSVOptions, ( + binary => 1, + sep_char => ';', + quote_char => '`', + escape_char => '`', + ) ); + +=head2 Operations before Create or Update + +The importer provides a callback to run operations before a ticket has been +created or updated from CSV content. To run some code before an update, add +the following to your CSV configuration file: + + Set($PreTicketChangeCallback, + sub { + my %args = ( + TicketObj => undef, + Row => undef, + Type => undef, + CurrentUser => undef, + @_, + ); + return 1; # to continue processing current row + } + ); + +As shown, you receive the ticket object(only for "Update" type), the current +CSV row, and the type of update, "Create" or "Update". CurrentUser is also +passed as it may be needed to call other methods. You can run any code in +the callback. + +The Row argument is a reference to a hash with the values from the CSV +file. The keys are the columns from the file and match the CSV +import configuration. The values are for the row currently being +processed. + +Since the Row argument is a reference, you can modify the value +before it is processed. For example, to lower case incoming status +values, you could do this: + + if ( exists $args{'Row'}->{status} ) { + $args{'Row'}->{status} = lc($args{'Row'}->{status}); + } + +If you return a false value, the change for that row is skipped, e.g. + + return ( 0, "Obsolete data" ); + +Return a true value to process that row normally. + + return 1; + +=head2 Operations after Create or Update + +The importer provides a callback to run operations after a ticket has been +created or updated from CSV content. To run some code after an update, add +the following to your CSV configuration file: + + Set($PostTicketChangeCallback, + sub { + my %args = ( + TicketObj => undef, + Row => undef, + Type => undef, + CurrentUser => undef, + @_, + ); + } + ); + +As shown, you receive the ticket object, the current CSV row, +and the type of update, "Create" or "Update". CurrentUser is also passed +as it may be needed to call other methods. You can run any code +in the callback. It expects no return value. + +=head2 Special Columns + +=over + +=item Comment or Correspond + +To add a comment or correspond (reply) to a ticket, you can map a CSV column +to "Comment" or "Correspond". When creating a ticket (--insert) you can use +either one and the content will be added to the Create transaction. + +=back + +=head2 TicketsImportTicketIdField + +If the CSV data contains the ids of existing RT tickets, you can set this option +to the name of the column containing the RT ticket id. The importer will then +search for that ticket id and update the ticket data with CSV values. + + Set($TicketsImportTicketIdField, 'RT ticket id'); + +Only one of TicketsImportTicketIdField or @TicketsImportUniqueCFs can be used +for a given CSV file. Also, this option is only valid for --update or --insert-update +modes. You cannot specify the ticket id to be created in --insert mode. + +=head2 TicketTolerantRoles + +By default, if a user can't be loaded via LDAP for a role, like Owner, +the importer will log it and skip creating the ticket. For roles that do not +require a successfully loaded user, set this option with the role name. +The importer will then log the failed attempt to find the user, but still +create the ticket. + + Set(@TicketTolerantRoles, 'CR.Subscribers Peers'); + +=head2 TransactionsImportFieldMapping + +Set the column mappings for importing transactions from a CSV file. A 'TicketID' mapping +is required for RT to add the transaction to an existing ticket. The 'TicketID' value is +mapped to the custom field 'Original Ticket ID'. + +Attachments can be included by providing the file system path for an attachment. + + Set( %TransactionsImportFieldMapping, + 'Attachment' => 'Attachment', + 'TicketID' => 'SomeID', + 'Created' => 'Date', + 'Type' => 'Type', + 'Content' => 'Content', + 'AttachmentType' => 'FileType' + ); + + +=cut + +=head1 AUTHOR + +Best Practical Solutions, LLC Emodules@bestpractical.comE + +=for html

All bugs should be reported via email to bug-RT-Extension-Import-CSV@rt.cpan.org +or via the web at rt.cpan.org.

+ +=for text + All bugs should be reported via email to + bug-RT-Extension-Import-CSV@rt.cpan.org + or via the web at + http://rt.cpan.org/Public/Dist/Display.html?Name=RT-Extension-Import-CSV + +=head1 LICENSE AND COPYRIGHT + +This software is Copyright (c) 2021 by Best Practical LLC + +This is free software, licensed under: + + The GNU General Public License, Version 2, June 1991 + +=cut + +1;