diff --git a/diff.c b/diff.c
index 2e33760439e397..8b039225b022fb 100644
--- a/diff.c
+++ b/diff.c
@@ -4835,6 +4835,8 @@ void repo_diff_setup(struct repository *r, struct diff_options *options)
 
 	options->color_moved = diff_color_moved_default;
 	options->color_moved_ws_handling = diff_color_moved_ws_default;
+
+	options->flags.allow_autoencode = -1;
 }
 
 static const char diff_status_letters[] = {
@@ -5029,6 +5031,9 @@ void diff_setup_done(struct diff_options *options)
 			options->filter = ~filter_bit[DIFF_STATUS_FILTER_AON];
 		options->filter &= ~options->filter_not;
 	}
+
+	if (options->flags.allow_autoencode == -1)
+		options->flags.allow_autoencode = !options->flags.binary;
 }
 
 int parse_long_opt(const char *opt, const char **argv,
diff --git a/diff.h b/diff.h
index 9ee7223f1da2c1..b270e8c1e64e17 100644
--- a/diff.h
+++ b/diff.h
@@ -186,7 +186,7 @@ struct diff_flags {
 	unsigned dirstat_by_file;
 	unsigned allow_textconv;
 	unsigned textconv_set_via_cmdline;
-	unsigned allow_autoencode;
+	int allow_autoencode;
 	unsigned diff_from_contents;
 	unsigned dirty_submodules;
 	unsigned ignore_untracked_in_submodules;