Merge pull request #2 from cassidylaidlaw/pypi-package

Configure PyPI package

LICENSE.txt

@@ -0,0 +1,17 @@
+MIT License
+Copyright (c) 2018 YOUR NAME
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,23 +1,30 @@
 # ReColorAdv
 
-This is an implementation of the ReColorAdv adversarial attack described in the NeurIPS 2019 paper ["Functional Adversarial Attacks"](https://arxiv.org/abs/1906.00001).
+This is an implementation of the ReColorAdv adversarial attack and other attacks described in the NeurIPS 2019 paper ["Functional Adversarial Attacks"](https://arxiv.org/abs/1906.00001).
 
 ## Getting Started
 
-ReColorAdv depends on the [`mister_ed`](https://github.com/revbucket/mister_ed) library. To install, navigate to the `ReColorAdv` directory and run
+Clone this repository by running
 
-    git clone https://github.com/revbucket/mister_ed.git
-    cd mister_ed
-    pip install -r requirements.txt
-    python scripts/setup_cifar.py
+    git clone https://github.com/cassidylaidlaw/ReColorAdv
 
-This will also download the CIFAR-10 dataset and pretrained models for it. Once `mister_ed` is installed, you can experiment with the ReColorAdv attack, by itself and combined with other attacks, in the `getting_started.ipynb` Jupyter notebook.
+You can experiment with the ReColorAdv attack, by itself and combined with other attacks, in the [`getting_started.ipynb`](getting_started.ipynb) Jupyter notebook.
 
-## Example Evaluation Script
+You can also install the ReColorAdv package with pip by running
 
-The script `evaluate_imagenet.py` will download a ResNet-50 trained on ImageNet and evaluate it against the ReColorAdv attack:
+    pip install recoloradv
 
-    python evaluate_imagenet.py --imagenet_path /path/to/ILSVRC2012 --batch_size 50
+## Evaluation Script (CIFAR-10)
+
+The script [`evaluate_cifar10.py`](recoloradv/examples/evaluate_cifar10.py) will evaluate a model trained on CIFAR-10 against the adversarial attacks in Table 1 of the paper. For instance, to evaluate a CIFAR-10 model trained on delta (L-infinity) attacks against a ReColorAdv+delta attack, run
+
+    python recoloradv/examples/evaluate_cifar10.py --checkpoint pretrained_models/delta.resnet32.pt --attack recoloradv+delta
+
+## Evaluation Script (ImageNet)
+
+The script [`evaluate_imagenet.py`](recoloradv/examples/evaluate_imagenet.py) will download a ResNet-50 trained on ImageNet and evaluate it against the ReColorAdv attack:
+
+    python recoloradv/examples/evaluate_imagenet.py --imagenet_path /path/to/ILSVRC2012 --batch_size 50
 
 ## Citation
 

getting_started.ipynb

@@ -24,12 +24,11 @@
    "source": [
     "# EXTERNAL LIBRARIES\n",
     "import numpy as np \n",
-    "import scipy \n",
+    "import re\n",
     "\n",
     "import torch\n",
     "import torch.nn as nn \n",
-    "import torch.optim as optim \n",
-    "assert float(torch.__version__[:3]) >= 0.3"
+    "import torch.optim as optim "
    ]
   },
   {
@@ -38,53 +37,25 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Add local libraries to the pythonpath\n",
-    "import os\n",
-    "import sys \n",
-    "module_path = os.path.abspath('mister_ed')\n",
-    "if module_path not in sys.path:\n",
-    "    sys.path.append(module_path)\n",
-    "\n",
     "# mister_ed\n",
-    "import config\n",
-    "import prebuilt_loss_functions as plf\n",
-    "import loss_functions as lf \n",
-    "import utils.pytorch_utils as utils\n",
-    "import utils.image_utils as img_utils\n",
-    "import cifar10.cifar_loader as cifar_loader\n",
-    "import cifar10.cifar_resnets as cifar_resnets\n",
-    "import adversarial_training as advtrain\n",
-    "import adversarial_evaluation as adveval\n",
-    "import utils.checkpoints as checkpoints\n",
-    "import adversarial_perturbations as ap \n",
-    "import adversarial_attacks as aa\n",
-    "import spatial_transformers as st\n",
+    "import recoloradv.mister_ed.loss_functions as lf \n",
+    "import recoloradv.mister_ed.utils.pytorch_utils as utils\n",
+    "import recoloradv.mister_ed.utils.image_utils as img_utils\n",
+    "import recoloradv.mister_ed.cifar10.cifar_loader as cifar_loader\n",
+    "import recoloradv.mister_ed.cifar10.cifar_resnets as cifar_resnets\n",
+    "import recoloradv.mister_ed.adversarial_training as advtrain\n",
+    "import recoloradv.mister_ed.utils.checkpoints as checkpoints\n",
+    "import recoloradv.mister_ed.adversarial_perturbations as ap \n",
+    "import recoloradv.mister_ed.adversarial_attacks as aa\n",
+    "import recoloradv.mister_ed.spatial_transformers as st\n",
+    "import recoloradv.mister_ed.config as config\n",
     "\n",
     "# ReColorAdv\n",
-    "import perturbations as pt\n",
-    "import color_transformers as ct\n",
-    "import color_spaces as cs\n",
-    "import norms"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Now let's make sure that you have CIFAR-10 data loaded and a pretrained classifier running. If the following block fails, then make sure you've run the setup script: from the `mister_ed` directory, run \n",
-    "\n",
-    "``` python scripts/setup_cifar.py```"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Quick check to ensure cifar 10 data and pretrained classifiers are loaded \n",
-    "cifar_valset = cifar_loader.load_cifar_data('val')\n",
-    "model, normalizer = cifar_loader.load_pretrained_cifar_resnet(flavor=32, return_normalizer=True)"
+    "import recoloradv.perturbations as pt\n",
+    "import recoloradv.color_transformers as ct\n",
+    "import recoloradv.color_spaces as cs\n",
+    "from recoloradv import norms\n",
+    "from recoloradv.utils import load_pretrained_cifar10_model, get_attack_from_name"
    ]
   },
   {
@@ -117,7 +88,7 @@
     "cifar_valset = cifar_loader.load_cifar_data('val', batch_size=16)\n",
     "examples, labels = next(iter(cifar_valset))\n",
     "\n",
-    "model, normalizer = cifar_loader.load_pretrained_cifar_resnet(flavor=32, return_normalizer=True)\n",
+    "model, normalizer = load_pretrained_cifar10_model('pretrained_models/normal.resnet32.pt')\n",
     "\n",
     "if utils.use_gpu():\n",
     "    examples = examples.cuda()\n",
@@ -254,13 +225,55 @@
    "source": [
     "Note that the resulting adversarial examples have been both recolored using ReColorAdv and had some additive adversarial noise applied from the delta attack."
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Prebuilt Attacks\n",
+    "The convenience function `get_attack_from_name` allows you to easily instantiate one of the attacks used in the paper. For instance, to use the combined ReColorAdv, StAdv, and delta attacks:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "attack = get_attack_from_name('recoloradv+stadv+delta', model, normalizer, verbose=True)\n",
+    "perturbation = attack.attack(examples, labels)\n",
+    "\n",
+    "# Display the successful adversarial examples.\n",
+    "successful_advs, successful_origs = perturbation.collect_successful(model, normalizer)\n",
+    "successful_diffs = ((successful_advs - successful_origs) * 3 + 0.5).clamp(0, 1)\n",
+    "img_utils.show_images([successful_origs, successful_advs, successful_diffs])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
    "display_name": "Python 3.7",
    "language": "python",
    "name": "python3.7"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
   }
  },
  "nbformat": 4,

pretrained_models/README.md

@@ -0,0 +1,8 @@
+# Pretrained models
+
+This directory contains pretrained models for CIFAR-10 from the paper. All models use the ResNet-32 architecture in `recoloradv.mister_ed.cifar10.cifar_resnets`. The pretrained models include
+ * A normally-trained model: [normal.resnet32.pt](normal.resnet32.pt).
+ * A model trained on black-and-white images: [bw.resnet32.pt](bw.resnet32.pt).
+ * Models trained on combinations of ReColorAdv, StAdv, and delta (L-infinity) attacks: [delta.resnet32.pt](delta.resnet32.pt), [stadv.resnet32.pt](stadv.resnet32.pt), [recoloradv.resnet32.pt](recoloradv.resnet32.pt), [stadv_delta.resnet32.pt](stadv_delta.resnet32.pt), [recoloradv_stadv.resnet32.pt](recoloradv_stadv.resnet32.pt), [recoloradv_delta.resnet32.pt](recoloradv_delta.resnet32.pt), and [recoloradv_stadv_delta.resnet32.pt](recoloradv_stadv_delta.resnet32.pt).
+
+

color_spaces.py → recoloradv/color_spaces.py

@@ -4,10 +4,10 @@
 
 import torch
 import torch.nn as nn
-import utils.pytorch_utils as utils
+from .mister_ed.utils import pytorch_utils as utils
 from torch.autograd import Variable
 import numpy as np
-import norms
+from recoloradv import norms
 import math
 
 
@@ -34,7 +34,7 @@ def to_rgb(self, imgs):
 class RGBColorSpace(ColorSpace):
     """
     RGB color space. Just applies identity transformation.
-    """
+    """ 
 
     def from_rgb(self, imgs):
         return imgs

color_transformers.py → recoloradv/color_transformers.py

@@ -4,9 +4,9 @@
 
 import torch
 import torch.nn as nn
-import utils.pytorch_utils as utils
+from .mister_ed.utils import pytorch_utils as utils
 from torch.autograd import Variable
-import norms
+from recoloradv import norms
 from functools import lru_cache
 
 

recoloradv/examples/evaluate_cifar10.py

@@ -0,0 +1,83 @@
+
+import torch
+import argparse
+import sys
+import os
+from torch import optim
+from torch.utils.data import DataLoader
+from torchvision.models import resnet50
+from torchvision.datasets import ImageNet
+from torchvision import transforms
+
+# mister_ed
+from recoloradv.mister_ed import loss_functions as lf
+from recoloradv.mister_ed import adversarial_training as advtrain
+from recoloradv.mister_ed import adversarial_perturbations as ap 
+from recoloradv.mister_ed import adversarial_attacks as aa
+from recoloradv.mister_ed import spatial_transformers as st
+from recoloradv.mister_ed.utils import pytorch_utils as utils
+from recoloradv.mister_ed.cifar10 import cifar_loader
+
+# ReColorAdv
+from recoloradv import perturbations as pt
+from recoloradv import color_transformers as ct
+from recoloradv import color_spaces as cs
+from recoloradv.utils import get_attack_from_name, load_pretrained_cifar10_model
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Evaluate a model trained on CIFAR-10 '
+        'against ReColorAdv and other attacks'
+    )
+
+    parser.add_argument('--checkpoint', type=str,
+                        help='checkpoint to evaluate')
+    parser.add_argument('--attack', type=str,
+                        help='attack to run, such as "recoloradv" or '
+                        '"stadv+delta"')
+    parser.add_argument('--batch_size', type=int, default=100,
+                        help='number of examples/minibatch')
+    parser.add_argument('--num_batches', type=int, required=False,
+                        help='number of batches (default entire dataset)')
+    args = parser.parse_args()
+
+    model, normalizer = load_pretrained_cifar10_model(args.checkpoint)
+    val_loader = cifar_loader.load_cifar_data('val', batch_size=args.batch_size)
+
+    model.eval()
+    if torch.cuda.is_available():
+        model.cuda()
+
+    attack = get_attack_from_name(args.attack, model, normalizer)
+
+    batches_correct = []
+    for batch_index, (inputs, labels) in enumerate(val_loader):
+        if (
+            args.num_batches is not None and
+            batch_index >= args.num_batches
+        ):
+            break
+
+        if torch.cuda.is_available():
+            inputs = inputs.cuda()
+            labels = labels.cuda()
+
+        adv_inputs = attack.attack(
+            inputs,
+            labels,
+        )[0]
+        with torch.no_grad():
+            adv_logits = model(normalizer(adv_inputs))
+        batch_correct = (adv_logits.argmax(1) == labels).detach()
+
+        batch_accuracy = batch_correct.float().mean().item()
+        print(f'BATCH {batch_index:05d}',
+              f'accuracy = {batch_accuracy * 100:.1f}',
+              sep='\t')
+        batches_correct.append(batch_correct)
+
+    accuracy = torch.cat(batches_correct).float().mean().item()
+    print('OVERALL    ',
+          f'accuracy = {accuracy * 100:.1f}',
+          sep='\t')

evaluate_imagenet.py → recoloradv/examples/evaluate_imagenet.py

@@ -9,20 +9,18 @@
 from torchvision.datasets import ImageNet
 from torchvision import transforms
 
-sys.path.append(os.path.abspath('mister_ed'))
-
 # mister_ed
-import loss_functions as lf
-import adversarial_training as advtrain
-import adversarial_perturbations as ap 
-import adversarial_attacks as aa
-import spatial_transformers as st
-import utils.pytorch_utils as utils
+from recoloradv.mister_ed import loss_functions as lf
+from recoloradv.mister_ed import adversarial_training as advtrain
+from recoloradv.mister_ed import adversarial_perturbations as ap 
+from recoloradv.mister_ed import adversarial_attacks as aa
+from recoloradv.mister_ed import spatial_transformers as st
+from recoloradv.mister_ed.utils import pytorch_utils as utils
 
 # ReColorAdv
-import perturbations as pt
-import color_transformers as ct
-import color_spaces as cs
+from recoloradv import perturbations as pt
+from recoloradv import color_transformers as ct
+from recoloradv import color_spaces as cs
 
 
 if __name__ == '__main__':
@@ -33,7 +31,7 @@
 
     parser.add_argument('--imagenet_path', type=str, required=True,
                         help='path to ImageNet dataset')
-    parser.add_argument('--batch_size', type=int, default=100, required=True,
+    parser.add_argument('--batch_size', type=int, default=100,
                         help='number of examples/minibatch')
     parser.add_argument('--num_batches', type=int, required=False,
                         help='number of batches (default entire dataset)')

recoloradv/mister_ed/README.md

@@ -0,0 +1 @@
+Code in this directory is adapted from the [`mister_ed`](https://github.com/revbucket/mister_ed) library.