diff --git a/.gitignore b/.gitignore index ddc5d4274..b715d85d8 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,4 @@ Thumbs.db # server private key file server/rematch/.rematch_secret.key server/postgres-data/ +release/dist/ diff --git a/.travis.yml b/.travis.yml index ba22b391d..79ba79ea7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,9 @@ matrix: services: - docker - python: "2.7" - env: PROJECT=setup.py + env: PROJECT=release - python: "3.6" - env: PROJECT=setup.py + env: PROJECT=release - python: "3.6" env: PROJECT=docs - python: "2.7" @@ -98,8 +98,8 @@ script: py.test -rapP ./${PROJECT} ./tests/${PROJECT} --verbose --cov-report= --cov=${PROJECT} ; fi ; fi ; - - if [ "${PROJECT}" == "setup.py" ]; then python ./setup.py server install ; fi ; - - if [ "${PROJECT}" == "setup.py" ]; then python ./setup.py idaplugin install ; fi ; + - if [ "${PROJECT}" == "release" ]; then python ./release server install ; fi ; + - if [ "${PROJECT}" == "release" ]; then python ./release idaplugin install ; fi ; - if [ "${PROJECT}" == "docs" ]; then sphinx-build ./docs/ ./docs/_build -a -E -n -W ; fi ; after_script: diff --git a/release/__init__.py b/release/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/release/package.py b/release/package.py new file mode 100644 index 000000000..9d3dcf426 --- /dev/null +++ b/release/package.py @@ -0,0 +1,89 @@ +import os +import re + +from pkg_resources import parse_version +from setuptools import setup, find_packages +from twine.cli import dispatch as twine + + +class Package(object): + def __init__(self, name, path, version_path, zip_safe, package_data=None, + classifiers=[]): + self.name = name + self.path = path + self.version_path = os.path.join(self.path, version_path, "version.py") + self.zip_safe = zip_safe + self.classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)" + ] + self.classifiers += classifiers + self.package_data = package_data or {} + + def get_version(self): + version_content = open(self.version_path).read() + # grab version string from file, this excludes inline comments too + version_str = re.search(r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', + version_content).group(1) + return parse_version(version_str) + + def get_released_version(self): + # TODO + pass + + def generate_changelog(self): + # TODO + pass + + def get_requirements(self, *parts): + fpath = os.path.join(*parts) + if not os.path.exists(fpath): + return [] + + with open(fpath) as fh: + return (l for l in fh.readlines() if not l.startswith('-r ')) + + def build(self, *script_args): + # generate install_requires based on requirements.txt + install_requires = self.get_requirements(self.path, "requirements.txt") + + # include requirementst.txt as part of package + if install_requires: + if self.path not in self.package_data: + self.package_data[self.path] = [] + self.package_data[self.path].append('requirements.txt') + + extras_require = {'test': self.get_requirements("tests", self.path, + "requirements.txt")} + + with open("README.rst") as fh: + long_description = fh.read() + + setup( + script_args=script_args + ('--dist-dir=./release/dist',), + name=self.name, + version=str(self.get_version()), + author="Nir Izraeli", + author_email="nirizr@gmail.com", + description=("A IDA Pro plugin and server framework for binary " + "function level diffing."), + keywords=["rematch", "ida", "idapro", "binary diffing", + "reverse engineering"], + url="https://www.github.com/nirizr/rematch/", + packages=find_packages(self.path), + package_dir={'': self.path}, + package_data=self.package_data, + extras_require=extras_require, + install_requires=install_requires, + long_description=long_description, + classifiers=self.classifiers + ) + + def get_dist_file(self): + return './release/dist/{}-{}.zip'.format(self.name, self.get_version()) + + def upload(self, repo="pypi"): + twine(['upload', self.get_dist_file(), '-r', repo]) + + def __repr__(self): + return "".format(self.name, self.get_version()) diff --git a/release/packages.py b/release/packages.py new file mode 100644 index 000000000..4be2366ec --- /dev/null +++ b/release/packages.py @@ -0,0 +1,21 @@ +from .package import Package + + +server = Package(name='rematch-server', path='server', version_path='./', + classifiers=["Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Environment :: Web Environment", + "Framework :: Django"], zip_safe=True) + +ida = Package(name='rematch-idaplugin', path='idaplugin', + version_path='rematch', zip_safe=False, + package_data={'idaplugin/rematch': ['images/*']}, + classifiers=["Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7"]) + +package_list = [server, ida] diff --git a/release/release.py b/release/release.py new file mode 100644 index 000000000..b99e86ee6 --- /dev/null +++ b/release/release.py @@ -0,0 +1,73 @@ +#!python + +import logging +import subprocess +import argparse + +from . import setup +from .packages import package_list + + +REMOTE="origin" +BRANCH="master" +# TODO: Remove next line just before merging +BRANCH="nirizr/release_py" + + +def sh_exec(*args): + logging.getLogger('sh_exec').info("Executing '%s'", args) + output = subprocess.check_output(args, shell=False).strip().decode() + logging.getLogger('sh_exec').info("Output '%s'", output) + return output + + +def validate_git_state(): + logging.info("Validating git state is clean") + + if sh_exec("git", "rev-parse", "--abbrev-ref", "HEAD") != BRANCH: + raise RuntimeError("Current branch name doesn't match release branch.") + + if "nothing to commit" not in sh_exec("git", "status", "-uno"): + raise RuntimeError("Local branch is dirty, can only release in clean " + "workspaces.") + + remote_branch = sh_exec("git", "ls-remote", REMOTE, "-h", + "refs/heads/" + BRANCH) + remote_branch_hash = remote_branch.split()[0] + if sh_exec("git", "rev-parse", BRANCH) is not remote_branch_hash: + raise RuntimeError("Local and remote branches are out of sync, releases " + "are only possible on up-to-date branch") + + +def identify_new_packages(): + new_packages = set() + + for package in package_list: + print(package) + # if package.get_version() + new_packages.add(package) + + return new_packages + + +def main(): + parser = argparse.ArgumentParser(description="Rematch release utility") + parser.add_argument('--verbose', '-v', action='count') + parser.add_argument('--skip-validation', '-sv', default=False, action='store_true') + args = parser.parse_args() + + logging.basicConfig(level=logging.ERROR - args.verbose * 10) + + if not args.skip_validation: + validate_git_state() + + packages = identify_new_packages() + for package in packages: + package.generate_changelog() + package.build('sdist', '--formats=zip') + package.upload('test') + package.upload() + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py deleted file mode 100755 index 61bd451d7..000000000 --- a/setup.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/python - -import sys -import os -from setuptools import setup, find_packages -import re - - -# Utility function to read the README file. -# Used for the long_description. It's nice, because now 1) we have a top level -# README file and 2) it's easier to type in the README file than to put a raw -# string in below ... -def read(fname): - return open(fname).read() - - -def get_version(path): - version_path = os.path.join(path, 'version.py') - return re.search( - r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', # It excludes inline comment too - open(version_path).read()).group(1) - - -def get_requirements(fname): - return (l for l in open(fname).readlines() if not l.startswith('-r ')) - - -def build_setup(package_base, package_name, version_path, - package_data=None, script_args=None): - if package_data is None: - package_data = {} - - # generate install_requires based on requirements.txt - base_path = os.path.abspath(os.path.dirname(__file__)) - requirements_path = os.path.join(base_path, package_base, "requirements.txt") - if os.path.exists(requirements_path): - install_requires = get_requirements(requirements_path) - # include requirementst.txt as part of package - if package_base not in package_data: - package_data[package_base] = [] - package_data[package_base].append('requirements.txt') - else: - install_requires = [] - - test_requirements_path = os.path.join(base_path, "tests", package_base, - "requirements.txt") - extras_require = {} - if os.path.exists(test_requirements_path): - extras_require['test'] = get_requirements(test_requirements_path) - - version_path = os.path.join(base_path, package_base, version_path) - readme_path = os.path.join(base_path, "README.rst") - setup( - script_args=script_args, - name=package_name, - version=get_version(version_path), - author="Nir Izraeli", - author_email="nirizr@gmail.com", - description=("A IDA Pro plugin and server framework for binary function " - "level diffing."), - keywords=["rematch", "ida", "idapro", "bindiff", "binary diffing", - "reverse engineering"], - url="https://www.github.com/nirizr/rematch/", - packages=find_packages(package_base), - package_dir={'': package_base}, - package_data=package_data, - extras_require=extras_require, - install_requires=install_requires, - long_description=read(readme_path), - classifiers=[ - "Development Status :: 3 - Alpha", - ], - ) - - -def build_setup_server(script_args=None): - build_setup(package_base='server', - package_name='rematch-server', - version_path='./', - script_args=script_args) - - -def build_setup_idaplugin(script_args=None): - package_data = {'idaplugin/rematch': ['images/*']} - build_setup(package_base='idaplugin', - package_name='rematch-idaplugin', - version_path='rematch', - package_data=package_data, - script_args=script_args) - - -if __name__ == '__main__': - expected_packages = {'server', 'idaplugin'} - packages = set(os.listdir('.')) & expected_packages - - if len(sys.argv) < 2 and len(packages) > 1: - print("Usage: {} {{package name}}".format(sys.argv[0])) - print("Available packages are: {}".format(", ".join(packages))) - sys.exit(1) - - # If all packages are available, allow a 'release' command that would push - # all packages to pypi - if sys.argv[1] == 'release' and packages == expected_packages: - script_args = ['sdist', '--dist-dir=./dist', '--formats=zip', 'upload'] - if not (len(sys.argv) >= 3 and sys.argv[2] == 'official'): - script_args += ['-r', 'pypitest'] - build_setup_server(script_args=script_args) - build_setup_idaplugin(script_args=script_args) - else: - package = packages.pop() if len(packages) == 1 else sys.argv[1] - if sys.argv[1] == package: - sys.argv = sys.argv[:1] + sys.argv[2:] - if package == 'server': - build_setup_server() - elif package == 'idaplugin': - build_setup_idaplugin()