Skip to content

Commit

Permalink
feat: apply global options to both concat and wrap commands
Browse files Browse the repository at this point in the history
  • Loading branch information
@ronaldtse
ronaldtse committed Jul 22, 2024
1 parent f3049e6 commit aaa63a9
Show file tree
Hide file tree
Showing 9 changed files with 329 additions and 246 deletions.
23 changes: 18 additions & 5 deletions README.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,24 @@ Commands:
poepod wrap GEMSPEC_PATH # Wrap a gem based on its gemspec file
----

=== Global options

All options can be used for both `wrap` and `concat` commands:

* `--exclude`: List of patterns to exclude (default: `["node_modules/", ".git/", ".gitignore$", ".DS_Store$", "^\\..+"]`)
* `--config`: Path to configuration file
* `--include-binary`: Include binary files (encoded in MIME format)
* `--include-dot-files`: Include dot files
* `--output-file`: Output path
* `--base-dir`: Base directory for relative file paths in output
* `--include-unstaged`: Include unstaged files from `lib`, `spec`, and `test` directories (for `wrap` command only)

[source,shell]
----
$ poepod concat FILES [OUTPUT_FILE] --exclude PATTERNS --config PATH --include-binary --include-dot-files --output-file PATH --base-dir PATH
$ poepod wrap GEMSPEC_PATH --exclude PATTERNS --config PATH --include-binary --include-dot-files --output-file PATH --base-dir PATH --include-unstaged
----

=== Concatenating files

The `concat` command allows you to combine multiple files into a single text
Expand All @@ -54,7 +72,6 @@ coding assistants.
By default, it excludes binary files, dot files, and certain patterns like
`node_modules/` and `.git/`.


[source,shell]
----
$ poepod concat path/to/files/* output.txt
Expand All @@ -63,7 +80,6 @@ $ poepod concat path/to/files/* output.txt
This will concatenate all non-binary, non-dot files from the specified path into
`output.txt`.


==== Including dot files

By default, dot files (hidden files starting with a dot) are excluded.
Expand All @@ -75,7 +91,6 @@ To include them, use the `--include-dot-files` option:
$ poepod concat path/to/files/* output.txt --include-dot-files
----


==== Including binary files

By default, binary files are excluded to keep the output focused on readable
Expand All @@ -92,7 +107,6 @@ $ poepod concat path/to/files/* output.txt --include-binary
This can be useful when you need to include binary assets or compiled files in
your analysis.


==== Excluding patterns

You can exclude certain patterns using the `--exclude` option:
Expand All @@ -105,7 +119,6 @@ $ poepod concat path/to/files/* output.txt --exclude node_modules .git build tes
This is helpful when you want to focus on specific parts of your codebase,
excluding irrelevant or large directories.


=== Wrapping a gem

The `wrap` command creates a comprehensive snapshot of your gem, including all
Expand Down
43 changes: 32 additions & 11 deletions lib/poepod/cli.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,31 +8,50 @@
module Poepod
# Command-line interface for Poepod
class Cli < Thor
# Define shared options
def self.shared_options
option :exclude, type: :array, default: Poepod::FileProcessor::EXCLUDE_DEFAULT,
desc: "List of patterns to exclude"
option :config, type: :string, desc: "Path to configuration file"
option :include_binary, type: :boolean, default: false, desc: "Include binary files (encoded in MIME format)"
option :include_dot_files, type: :boolean, default: false, desc: "Include dot files"
option :output_file, type: :string, desc: "Output path"
option :base_dir, type: :string, desc: "Base directory for relative file paths in output"
end

desc "concat FILES [OUTPUT_FILE]", "Concatenate specified files into one text file"
option :exclude, type: :array, default: Poepod::FileProcessor::EXCLUDE_DEFAULT, desc: "List of patterns to exclude"
option :config, type: :string, desc: "Path to configuration file"
option :include_binary, type: :boolean, default: false, desc: "Include binary files (encoded in MIME format)"
option :include_dot_files, type: :boolean, default: false, desc: "Include dot files"
option :output_file, type: :string, desc: "Output path"
shared_options

def concat(*files)
check_files(files)
output_file = determine_output_file(files)
process_files(files, output_file)
base_dir = options[:base_dir] || Dir.pwd
process_files(files, output_file, base_dir)
end

desc "wrap GEMSPEC_PATH", "Wrap a gem based on its gemspec file"
shared_options
option :include_unstaged, type: :boolean, default: false,
desc: "Include unstaged files from lib, spec, and test directories"

def wrap(gemspec_path)
base_dir = options[:base_dir] || File.dirname(gemspec_path)
processor = Poepod::GemProcessor.new(
gemspec_path,
nil,
include_unstaged: options[:include_unstaged]
include_unstaged: options[:include_unstaged],
exclude: options[:exclude],
include_binary: options[:include_binary],
include_dot_files: options[:include_dot_files],
base_dir: base_dir,
config_file: options[:config]
)
success, result, unstaged_files = processor.process
handle_wrap_result(success, result, unstaged_files)
if success
handle_wrap_result(success, result, unstaged_files)
else
puts result
exit(1)
end
end

def self.exit_on_failure?
Expand All @@ -52,14 +71,16 @@ def determine_output_file(files)
options[:output_file] || default_output_file(files.first)
end

def process_files(files, output_file)
def process_files(files, output_file, base_dir)
output_path = Pathname.new(output_file).expand_path
processor = Poepod::FileProcessor.new(
files,
output_path,
config_file: options[:config],
include_binary: options[:include_binary],
include_dot_files: options[:include_dot_files]
include_dot_files: options[:include_dot_files],
exclude: options[:exclude],
base_dir: base_dir
)
total_files, copied_files = processor.process
print_result(total_files, copied_files, output_path)
Expand Down
103 changes: 18 additions & 85 deletions lib/poepod/file_processor.rb
View file
Original file line number Diff line number Diff line change
@@ -1,43 +1,32 @@
# frozen_string_literal: true

require_relative "processor"
require "yaml"
require "tqdm"
require "pathname"
require "open3"
require "base64"
require "mime/types"

module Poepod
# Processes files for concatenation, handling binary and dot files
class FileProcessor < Processor
EXCLUDE_DEFAULT = [
%r{node_modules/}, %r{.git/}, /.gitignore$/, /.DS_Store$/, /^\..+/ # Add dot files pattern
%r{node_modules/}, %r{.git/}, /.gitignore$/, /.DS_Store$/, /^\..+/
].freeze

def initialize(files, output_file, config_file: nil, include_binary: false, include_dot_files: false)
super(config_file)
def initialize(
files,
output_file,
config_file: nil,
include_binary: false,
include_dot_files: false,
exclude: [],
base_dir: nil
)
super(
config_file,
include_binary: include_binary,
include_dot_files: include_dot_files,
exclude: exclude,
base_dir: base_dir,
)
@files = files
@output_file = output_file
@failed_files = []
@include_binary = include_binary
@include_dot_files = include_dot_files
end

def process
_ = 0
copied_files = 0
files_to_process = collect_files_to_process
total_files = files_to_process.size

File.open(@output_file, "w", encoding: "utf-8") do |output|
files_to_process.sort.each do |file_path|
process_single_file(file_path, output)
copied_files += 1
end
end

[total_files, copied_files]
end

private
Expand All @@ -46,67 +35,11 @@ def collect_files_to_process
@files.flatten.each_with_object([]) do |file, files_to_process|
Dir.glob(file, File::FNM_DOTMATCH).each do |matched_file|
next unless File.file?(matched_file)
next if dot_file?(matched_file) && !@include_dot_files
next if binary_file?(matched_file) && !@include_binary
next if should_exclude?(matched_file)

files_to_process << matched_file
end
end
end

def process_single_file(file_path, output)
file_path, content, error = process_file(file_path)
if content
output.puts "--- START FILE: #{file_path} ---"
output.puts content
output.puts "--- END FILE: #{file_path} ---"
elsif error
warn "ERROR: #{file_path}: #{error}"
end
end

def dot_file?(file_path)
File.basename(file_path).start_with?(".")
end

def binary_file?(file_path)
!text_file?(file_path)
end

def process_file(file_path)
if text_file?(file_path)
process_text_file(file_path)
elsif @include_binary
process_binary_file(file_path)
else
[file_path, nil, nil] # Skipped binary file
end
rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError
handle_encoding_error(file_path)
end

def process_text_file(file_path)
[file_path, File.read(file_path, encoding: "utf-8"), nil]
end

def process_binary_file(file_path)
[file_path, encode_binary_file(file_path), nil]
end

def handle_encoding_error(file_path)
@failed_files << file_path
[file_path, nil, "Failed to decode the file, as it is not saved with UTF-8 encoding."]
end

def text_file?(file_path)
stdout, status = Open3.capture2("file", "-b", "--mime-type", file_path)
status.success? && stdout.strip.start_with?("text/")
end

def encode_binary_file(file_path)
mime_type = MIME::Types.type_for(file_path).first.content_type
encoded_content = Base64.strict_encode64(File.binread(file_path))
"Content-Type: #{mime_type}\nContent-Transfer-Encoding: base64\n\n#{encoded_content}"
end
end
end
84 changes: 39 additions & 45 deletions lib/poepod/gem_processor.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,22 @@
module Poepod
# Processes gem files for wrapping, handling unstaged files
class GemProcessor < Processor
def initialize(gemspec_path, config_file = nil, include_unstaged: false)
super(config_file)
def initialize(
gemspec_path,
include_unstaged: false,
exclude: [],
include_binary: false,
include_dot_files: false,
base_dir: nil,
config_file: nil
)
super(
config_file,
include_binary: include_binary,
include_dot_files: include_dot_files,
exclude: exclude,
base_dir: base_dir || File.dirname(gemspec_path),
)
@gemspec_path = gemspec_path
@include_unstaged = include_unstaged
end
Expand All @@ -21,16 +35,31 @@ def process
return spec unless spec.is_a?(Gem::Specification)

gem_name = spec.name
output_file = "#{gem_name}_wrapped.txt"
@output_file = "#{gem_name}_wrapped.txt"
unstaged_files = check_unstaged_files

write_wrapped_gem(spec, output_file, unstaged_files)
super()

[true, output_file, unstaged_files]
[true, @output_file, unstaged_files]
end

private

def collect_files_to_process
spec = load_gemspec
files_to_include = (spec.files +
spec.test_files +
find_readme_files).uniq

files_to_include += check_unstaged_files if @include_unstaged

files_to_include.sort.uniq.reject do |relative_path|
should_exclude?(File.join(@base_dir, relative_path))
end.map do |relative_path|
File.join(@base_dir, relative_path)
end
end

def error_no_gemspec
[false, "Error: The specified gemspec file '#{@gemspec_path}' does not exist."]
end
Expand All @@ -41,47 +70,12 @@ def load_gemspec
[false, "Error loading gemspec: #{e.message}"]
end

def write_wrapped_gem(spec, output_file, unstaged_files)
File.open(output_file, "w") do |file|
write_header(file, spec)
write_unstaged_warning(file, unstaged_files) if unstaged_files.any?
write_files_content(file, spec, unstaged_files)
end
end

def write_header(file, spec)
file.puts "# Wrapped Gem: #{spec.name}"
file.puts "## Gemspec: #{File.basename(@gemspec_path)}"
end

def write_unstaged_warning(file, unstaged_files)
file.puts "\n## Warning: Unstaged Files"
file.puts unstaged_files.sort.join("\n")
file.puts "\nThese files are not included in the wrap unless --include-unstaged option is used."
end

def write_files_content(file, spec, unstaged_files)
file.puts "\n## Files:\n"
files_to_include = (spec.files + spec.test_files + find_readme_files).uniq
files_to_include += unstaged_files if @include_unstaged

files_to_include.sort.uniq.each do |relative_path|
write_file_content(file, relative_path)
end
end

def write_file_content(file, relative_path)
full_path = File.join(File.dirname(@gemspec_path), relative_path)
return unless File.file?(full_path)

file.puts "--- START FILE: #{relative_path} ---"
file.puts File.read(full_path)
file.puts "--- END FILE: #{relative_path} ---\n\n"
end

def find_readme_files
Dir.glob(File.join(File.dirname(@gemspec_path), "README*"))
.map { |path| Pathname.new(path).relative_path_from(Pathname.new(File.dirname(@gemspec_path))).to_s }
Dir.glob(File.join(File.dirname(@gemspec_path), "README*")).map do |path|
Pathname.new(path).relative_path_from(
Pathname.new(File.dirname(@gemspec_path))
).to_s
end
end

def check_unstaged_files
Expand Down
Loading

0 comments on commit aaa63a9

Please sign in to comment.