diff --git a/pattern.c b/pattern.c
index 86bcab8..2f80c24 100644
--- a/pattern.c
+++ b/pattern.c
@@ -1145,9 +1145,9 @@ static const unsigned char b58_case_map[256] = {
 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 1, 1, 1, 1, 1, 1, 1, 1, 105, 1, 1, 0, 1, 1, 111,
+	0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 0, 1, 1, 2,
 	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,
-	0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 76, 1, 1, 0,
+	0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 2, 1, 1, 0,
 	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,
 };
 
@@ -1162,13 +1162,16 @@ prefix_case_iter_init(prefix_case_iter_t *cip, const char *pfx)
 		if (i > sizeof(cip->ci_prefix))
 			return 0;
 		if (!b58_case_map[(int)pfx[i]]) {
+			/* Character isn't case-swappable, ignore it */
 			cip->ci_prefix[i] = pfx[i];
 			continue;
 		}
-		if (b58_case_map[(int)pfx[i]] > 1) {
-			cip->ci_prefix[i] = b58_case_map[(int)pfx[i]];
+		if (b58_case_map[(int)pfx[i]] == 2) {
+			/* Character invalid, but valid in swapped case */
+			cip->ci_prefix[i] = pfx[i] ^ 0x20;
 			continue;
 		}
+		/* Character is case-swappable */
 		cip->ci_prefix[i] = pfx[i] | 0x20;
 		cip->ci_case_map[(int)cip->ci_nbits] = i;
 		cip->ci_nbits++;