Utility function for accuracy and l2 norm [JATIC-I2-IBM]

art/metrics/metrics.py

@@ -23,7 +23,7 @@
 
 from functools import reduce
 import logging
-from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING
+from typing import Any, Dict, List, Tuple, Optional, Union, TYPE_CHECKING
 
 import numpy as np
 import numpy.linalg as la
@@ -480,3 +480,41 @@
  w_d[i] = scipy.stats.wasserstein_distance(u_values[i], v_values[i], u_weights[i], v_weights[i])
 
  return w_d
+
+def adversarial_accuracy_and_perturbation(
+ classifier: "CLASSIFIER_TYPE",
+ x: np.ndarray,
+ x_adv: np.ndarray,
+ y: np.ndarray,
+ norm_type: int = 2,
+) -> Tuple[float, float, float]:
+ """
+ Compute accuracy and perturbation
+
+ :param classifier: A trained model.
+ :param x: Input samples of shape that can be fed into `classifier`.
+ :param x_adv: Adversarial input samples of shape that can be fed into the `classifier`.
+ :param y: True labels of `x`.
+ :return: Tuple including:
+ - clean accuracy of the classifier computed on `x`
+ - clean accuracy of the classifier computed on `x`
+ - perturbation (distance) between x and x_adv, calculated using norm (2, inf)
+ """
+ y_orig = np.argmax(classifier.predict(x), axis=1)
+ y_pred = np.argmax(classifier.predict(x_adv), axis=1)
+
+ if len(y.shape) > 1:
+ y = np.argmax(y, axis=1)
+
+ y_corr = y_orig == y
+
+ clean_acc = np.sum(y_corr) / len(y_orig)
+ robust_acc = np.sum((y_pred == y_orig) & y_corr) / np.sum(y_corr)
+
+ idxs = y_pred != y
+ avg_perts = 0.0
+ perts_norm = la.norm((x_adv - x).reshape(x.shape[0], -1), ord=norm_type, axis=1)
+ perts_norm = perts_norm[idxs]
+ avg_perts = np.mean(perts_norm)
+
+ return (clean_acc, robust_acc, avg_perts)

notebooks/attack_parallel_auto_attack.ipynb

@@ -16,7 +16,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 1,
+ "execution_count": 2,
  "metadata": {},
  "outputs": [],
  "source": [
@@ -56,7 +56,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 2,
+ "execution_count": 3,
  "metadata": {},
  "outputs": [
  {
@@ -92,7 +92,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 3,
+ "execution_count": 4,
  "metadata": {},
  "outputs": [
  {
@@ -158,7 +158,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 4,
+ "execution_count": 5,
  "metadata": {},
  "outputs": [
  {
@@ -168,11 +168,7 @@
  "/Users/kieranfraser/git/personal/art/env/lib/python3.9/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 1 leaked semaphore objects to clean up at shutdown\n",
  " warnings.warn('resource_tracker: There appear to be %d '\n",
  "/Users/kieranfraser/git/personal/art/env/lib/python3.9/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 1 leaked semaphore objects to clean up at shutdown\n",
- " warnings.warn('resource_tracker: There appear to be %d '\n",
- "/Users/kieranfraser/git/personal/art/env/lib/python3.9/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 1 leaked semaphore objects to clean up at shutdown\n",
- " warnings.warn('resource_tracker: There appear to be %d '\n",
- "/Users/kieranfraser/git/personal/art/env/lib/python3.9/multiprocessing/resource_tracker.py:229: UserWarning: resource_tracker: '/loky-2595-nh_wgd4i': [Errno 2] No such file or directory\n",
- " warnings.warn('resource_tracker: %r: %s' % (name, e))\n"
+ " warnings.warn('resource_tracker: There appear to be %d '\n"
  ]
  }
  ],
@@ -194,7 +190,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 5,
+ "execution_count": 6,
  "metadata": {},
  "outputs": [
  {
@@ -226,6 +222,37 @@
  "- The fourth column images are adversarial images generated by AutoAttack in parallel mode. Note how parallel model achieves lower L2 distance between the original and adversarial image - as all jobs are run in parallel mode, more attacks can be evaluated and successful attacks with lower perturbations added can be selected."
  ]
  },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Parallel\n",
+ "clean acc: 0.5\n",
+ "robust acc: 0.0\n",
+ "l2: 1.2837049961090088\n",
+ "----\n",
+ "Non-parallel\n",
+ "clean acc: 0.5\n",
+ "robust acc: 0.0\n",
+ "l2: 1.6767956018447876\n"
+ ]
+ }
+ ],
+ "source": [
+ "from art.metrics.metrics import adversarial_accuracy_and_perturbation\n",
+ "\n",
+ "clean_accuracy, robust_accuracy, perturbation = adversarial_accuracy_and_perturbation(classifier, x_train, parallel_adv, y_train)\n",
+ "print(f'Parallel\\nclean acc: {clean_accuracy}\\nrobust acc: {robust_accuracy}\\nl2: {perturbation}')\n",
+ "print('----')\n",
+ "clean_accuracy, robust_accuracy, perturbation = adversarial_accuracy_and_perturbation(classifier, x_train, nonparallel_adv, y_train)\n",
+ "print(f'Non-parallel\\nclean acc: {clean_accuracy}\\nrobust acc: {robust_accuracy}\\nl2: {perturbation}')"
+ ]
+ },
  {
  "cell_type": "code",
  "execution_count": 6,