diff --git a/src/pl/plpgsql/src/pl_gram.y b/src/pl/plpgsql/src/pl_gram.y
index 225c62ab36f565d9008781294f97c72aa29cd2b5..a1ae1bbf2cded334ba9ec1cb64783cd0077e4b11 100644
--- a/src/pl/plpgsql/src/pl_gram.y
+++ b/src/pl/plpgsql/src/pl_gram.y
@@ -1676,11 +1676,11 @@ stmt_exit		: exit_type opt_label opt_exitcond
 							if (label == NULL)
 								ereport(ERROR,
 										(errcode(ERRCODE_SYNTAX_ERROR),
-										 errmsg("label \"%s\" does not exist",
+										 errmsg("there is no label \"%s\" surrounding this statement",
 												$2),
 										 parser_errposition(@2)));
 							/* CONTINUE only allows loop labels */
-							if (label->itemno != PLPGSQL_LABEL_LOOP && !$1)
+							if (label->itemno != PLPGSQL_LABEL_LOOP && !new->is_exit)
 								ereport(ERROR,
 										(errcode(ERRCODE_SYNTAX_ERROR),
 										 errmsg("block label \"%s\" cannot be used in CONTINUE",
@@ -1697,9 +1697,9 @@ stmt_exit		: exit_type opt_label opt_exitcond
 							if (plpgsql_ns_find_nearest_loop(plpgsql_ns_top()) == NULL)
 								ereport(ERROR,
 										(errcode(ERRCODE_SYNTAX_ERROR),
-										 /* translator: %s is EXIT or CONTINUE */
-										 errmsg("%s cannot be used outside a loop",
-												plpgsql_stmt_typename((PLpgSQL_stmt *) new)),
+										 new->is_exit ?
+										 errmsg("EXIT cannot be used outside a loop, unless it has a label") :
+										 errmsg("CONTINUE cannot be used outside a loop"),
 										 parser_errposition(@1)));
 						}
 
diff --git a/src/test/regress/expected/plpgsql.out b/src/test/regress/expected/plpgsql.out
index b10157bb170d7015f02dca22b9039a1930fa8615..c1822aadf0a872103c3fb3bdffdcfdec4f2d1863 100644
--- a/src/test/regress/expected/plpgsql.out
+++ b/src/test/regress/expected/plpgsql.out
@@ -2843,7 +2843,7 @@ $$ language plpgsql;
 ERROR:  CONTINUE cannot be used outside a loop
 LINE 4:         continue;
                 ^
--- should fail: EXIT is only legal inside a loop
+-- should fail: unlabeled EXIT is only legal inside a loop
 create function exit_error1() returns void as $$
 begin
     begin
@@ -2851,7 +2851,7 @@ begin
     end;
 end;
 $$ language plpgsql;
-ERROR:  EXIT cannot be used outside a loop
+ERROR:  EXIT cannot be used outside a loop, unless it has a label
 LINE 4:         exit;
                 ^
 -- should fail: no such label
@@ -2864,7 +2864,7 @@ begin
     end;
 end;
 $$ language plpgsql;
-ERROR:  label "no_such_label" does not exist
+ERROR:  there is no label "no_such_label" surrounding this statement
 LINE 5:             continue no_such_label;
                              ^
 -- should fail: no such label
@@ -2877,7 +2877,7 @@ begin
     end;
 end;
 $$ language plpgsql;
-ERROR:  label "no_such_label" does not exist
+ERROR:  there is no label "no_such_label" surrounding this statement
 LINE 5:             exit no_such_label;
                          ^
 -- should fail: CONTINUE can't reference the label of a named block
diff --git a/src/test/regress/sql/plpgsql.sql b/src/test/regress/sql/plpgsql.sql
index 7b4191ecf97ed8424a97b4aef169b26bed024b53..f883e7e0a21a60c431b9c8427804aacd486381ab 100644
--- a/src/test/regress/sql/plpgsql.sql
+++ b/src/test/regress/sql/plpgsql.sql
@@ -2373,7 +2373,7 @@ begin
 end;
 $$ language plpgsql;
 
--- should fail: EXIT is only legal inside a loop
+-- should fail: unlabeled EXIT is only legal inside a loop
 create function exit_error1() returns void as $$
 begin
     begin