Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_tfclassifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=tfc, targeted=True, max_iter=2, max_eval=100, init_eval=10)
params = {'y': random_targets(y_test, tfc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist')
x_train, y_train, x_test, y_test = x_train[:NB_TRAIN], y_train[:NB_TRAIN], x_test[:NB_TEST], y_test[:NB_TEST]
cls.mnist = (x_train, y_train), (x_test, y_test)
# Keras classifier
cls.classifier_k = get_classifier_kr()
scores = cls.classifier_k._model.evaluate(x_train, y_train)
logging.info('[Keras, MNIST] Accuracy on training set: %.2f%%', (scores[1] * 100))
scores = cls.classifier_k._model.evaluate(x_test, y_test)
logging.info('[Keras, MNIST] Accuracy on test set: %.2f%%', (scores[1] * 100))
# Create basic CNN on MNIST using TensorFlow
cls.classifier_tf, sess = get_classifier_tf()
scores = get_labels_np_array(cls.classifier_tf.predict(x_train))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(y_train, axis=1)) / y_train.shape[0]
logging.info('[TF, MNIST] Accuracy on training set: %.2f%%', (acc * 100))
scores = get_labels_np_array(cls.classifier_tf.predict(x_test))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0]
logging.info('[TF, MNIST] Accuracy on test set: %.2f%%', (acc * 100))
# Create basic PyTorch model
cls.classifier_py = get_classifier_pt()
x_train, x_test = np.swapaxes(x_train, 1, 3), np.swapaxes(x_test, 1, 3)
x_train, x_test = x_train.astype(np.float32), x_test.astype(np.float32)
scores = get_labels_np_array(cls.classifier_py.predict(x_train))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(y_train, axis=1)) / y_train.shape[0]
def test_tfclassifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Attack
up = UniversalPerturbation(tfc, max_iter=1, attacker="newtonfool", attacker_params={"max_iter": 5})
x_train_adv = up.generate(x_train)
self.assertTrue((up.fooling_rate >= 0.2) or not up.converged)
x_test_adv = x_test + up.noise
self.assertFalse((x_test == x_test_adv).all())
train_y_pred = np.argmax(tfc.predict(x_train_adv), axis=1)
test_y_pred = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == test_y_pred).all())
self.assertFalse((np.argmax(y_train, axis=1) == train_y_pred).all())
def test_tfclassifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# Attack
attack_st = SpatialTransformation(tfc, max_translation=10.0, num_translations=3, max_rotation=30.0,
num_rotations=3)
x_train_adv = attack_st.generate(self.x_train)
self.assertAlmostEqual(x_train_adv[0, 8, 13, 0], 0.49004024, delta=0.01)
self.assertAlmostEqual(attack_st.fooling_rate, 0.72, delta=0.01)
self.assertEqual(attack_st.attack_trans_x, 3)
self.assertEqual(attack_st.attack_trans_y, 3)
self.assertEqual(attack_st.attack_rot, 30.0)
x_test_adv = attack_st.generate(self.x_test)
self.assertAlmostEqual(x_test_adv[0, 14, 14, 0], 0.013572651, delta=0.01)
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc, sess = get_classifier_tf()
# First attack
clinfm = CarliniLInfMethod(classifier=krc, targeted=True, max_iter=10, eps=0.5)
params = {'y': random_targets(self.y_test, krc.nb_classes())}
x_test_adv = clinfm.generate(self.x_test, **params)
self.assertFalse((self.x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', target)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f', (np.sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target == y_pred_adv).any())
# Second attack
def test_predict(self):
classifier, sess = get_classifier_tf()
predictions = classifier.predict(self.x_test)
predictions_class = np.argmax(predictions, axis=1)
trues_class = np.argmax(self.y_test, axis=1)
accuracy = np.sum(predictions_class == trues_class) / len(trues_class)
logger.info('Accuracy after fitting: %.2f%%', (accuracy * 100))
self.assertEqual(accuracy, 0.4)
tf.reset_default_graph()
sess.close()
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist')
x_train, y_train, x_test, y_test = x_train[:NB_TRAIN], y_train[:NB_TRAIN], x_test[:NB_TEST], y_test[:NB_TEST]
cls.mnist = (x_train, y_train), (x_test, y_test)
# Keras classifier
cls.classifier_k, sess = get_classifier_kr()
scores = cls.classifier_k._model.evaluate(x_train, y_train)
logging.info('[Keras, MNIST] Accuracy on training set: %.2f%%', (scores[1] * 100))
scores = cls.classifier_k._model.evaluate(x_test, y_test)
logging.info('[Keras, MNIST] Accuracy on test set: %.2f%%', (scores[1] * 100))
# Create basic CNN on MNIST using TensorFlow
cls.classifier_tf, sess = get_classifier_tf()
scores = get_labels_np_array(cls.classifier_tf.predict(x_train))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(y_train, axis=1)) / y_train.shape[0]
logging.info('[TF, MNIST] Accuracy on training set: %.2f%%', (acc * 100))
scores = get_labels_np_array(cls.classifier_tf.predict(x_test))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0]
logging.info('[TF, MNIST] Accuracy on test set: %.2f%%', (acc * 100))
# Create basic PyTorch model
cls.classifier_py = get_classifier_pt()
x_train, x_test = np.swapaxes(x_train, 1, 3), np.swapaxes(x_test, 1, 3)
scores = get_labels_np_array(cls.classifier_py.predict(x_train))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(y_train, axis=1)) / y_train.shape[0]
logging.info('[PyTorch, MNIST] Accuracy on training set: %.2f%%', (acc * 100))
def test_nb_classes(self):
classifier, sess = get_classifier_tf()
self.assertEqual(classifier.nb_classes(), 10)
tf.reset_default_graph()
sess.close()
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist')
x_train, y_train, x_test, y_test = x_train[:NB_TRAIN], y_train[:NB_TRAIN], x_test[:NB_TEST], y_test[:NB_TEST]
cls.mnist = (x_train, y_train), (x_test, y_test)
# Keras classifier
cls.classifier_k = get_classifier_kr()
scores = cls.classifier_k._model.evaluate(x_train, y_train)
logger.info('[Keras, MNIST] Accuracy on training set: %.2f%%', (scores[1] * 100))
scores = cls.classifier_k._model.evaluate(x_test, y_test)
logger.info('[Keras, MNIST] Accuracy on test set: %.2f%%', (scores[1] * 100))
# Create basic CNN on MNIST using TensorFlow
cls.classifier_tf, sess = get_classifier_tf()
scores = get_labels_np_array(cls.classifier_tf.predict(x_train))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(y_train, axis=1)) / y_train.shape[0]
logger.info('[TF, MNIST] Accuracy on training set: %.2f%%', (acc * 100))
scores = get_labels_np_array(cls.classifier_tf.predict(x_test))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('[TF, MNIST] Accuracy on test set: %.2f%%', (acc * 100))
# Create basic PyTorch model
cls.classifier_py = get_classifier_pt()
x_train, x_test = np.swapaxes(x_train, 1, 3), np.swapaxes(x_test, 1, 3)
x_train, x_test = x_train.astype(np.float32), x_test.astype(np.float32)
scores = get_labels_np_array(cls.classifier_py.predict(x_train))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(y_train, axis=1)) / y_train.shape[0]
cls.x_train = x_train[:NB_TRAIN]
cls.y_train = y_train[:NB_TRAIN]
cls.x_test = x_test[:NB_TEST]
cls.y_test = y_test[:NB_TEST]
# Keras classifier
cls.classifier_k = get_classifier_kr()
scores = cls.classifier_k._model.evaluate(x_train, y_train)
logger.info('[Keras, MNIST] Accuracy on training set: %.2f%%', (scores[1] * 100))
scores = cls.classifier_k._model.evaluate(x_test, y_test)
logger.info('[Keras, MNIST] Accuracy on test set: %.2f%%', (scores[1] * 100))
# Create basic CNN on MNIST using TensorFlow
cls.classifier_tf, sess = get_classifier_tf()
scores = get_labels_np_array(cls.classifier_tf.predict(x_train))
accuracy = np.sum(np.argmax(scores, axis=1) == np.argmax(y_train, axis=1)) / y_train.shape[0]
logger.info('[TF, MNIST] Accuracy on training set: %.2f%%', (accuracy * 100))
scores = get_labels_np_array(cls.classifier_tf.predict(x_test))
accuracy = np.sum(np.argmax(scores, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('[TF, MNIST] Accuracy on test set: %.2f%%', (accuracy * 100))
# Create basic PyTorch model
cls.classifier_py = get_classifier_pt()
x_train, x_test = np.swapaxes(x_train, 1, 3), np.swapaxes(x_test, 1, 3)
scores = get_labels_np_array(cls.classifier_py.predict(x_train.astype(np.float32)))
accuracy = np.sum(np.argmax(scores, axis=1) == np.argmax(y_train, axis=1)) / y_train.shape[0]
logger.info('[PyTorch, MNIST] Accuracy on training set: %.2f%%', (accuracy * 100))