From f9f258281e0b3ccae1ca3e349f527e4b64c898ba Mon Sep 17 00:00:00 2001
From: Tom Lane <tgl@sss.pgh.pa.us>
Date: Sun, 30 Sep 2001 18:57:45 +0000
Subject: [PATCH] Create a GUC parameter max_files_per_process that is a
configurable upper limit on what we will believe from sysconf(_SC_OPEN_MAX).
The default value is 1000, so that under ordinary conditions it won't affect
the behavior. But on platforms where the kernel promises far more than it
can deliver, this can be used to prevent running out of file descriptors.
See numerous past discussions, eg, pgsql-hackers around 23-Dec-2000.
---
doc/src/sgml/runtime.sgml | 22 +++++++-
src/backend/storage/file/fd.c | 51 ++++++++++++++-----
src/backend/utils/misc/guc.c | 6 ++-
src/backend/utils/misc/postgresql.conf.sample | 5 +-
src/include/storage/fd.h | 7 ++-
5 files changed, 73 insertions(+), 18 deletions(-)
diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml
index 8cb1eb364ae..6afdd9c9637 100644
--- a/doc/src/sgml/runtime.sgml
+++ b/doc/src/sgml/runtime.sgml
@@ -1,5 +1,5 @@
<!--
-$Header: /cvsroot/pgsql/doc/src/sgml/runtime.sgml,v 1.85 2001/09/23 21:52:36 petere Exp $
+$Header: /cvsroot/pgsql/doc/src/sgml/runtime.sgml,v 1.86 2001/09/30 18:57:45 tgl Exp $
-->
<Chapter Id="runtime">
@@ -1218,6 +1218,26 @@ dynamic_library_path = '/usr/local/lib/postgresql:/home/my_project/lib:$libdir'
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><varname>MAX_FILES_PER_PROCESS</varname> (<type>integer</type>)</term>
+ <listitem>
+ <para>
+ Sets the maximum number of simultaneously open files in each server
+ process. The default is 1000. The limit actually used by the code
+ is the smaller of this setting and the result of
+ <literal>sysconf(_SC_OPEN_MAX)</literal>.
+ Therefore, on systems where sysconf returns a reasonable limit,
+ you don't need to worry about this setting. But on some platforms
+ (notably, most BSD systems), sysconf returns a value that is much
+ larger than the system can really support when a large number of
+ processes all try to open that many files. If you find yourself
+ seeing <quote>Too many open files</> failures, try reducing this
+ setting.
+ This option can only be set at server start.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><varname>MAX_FSM_RELATIONS</varname> (<type>integer</type>)</term>
<listitem>
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
index a696f1fa1fb..c2db3c8dea8 100644
--- a/src/backend/storage/file/fd.c
+++ b/src/backend/storage/file/fd.c
@@ -7,7 +7,7 @@
* Portions Copyright (c) 1994, Regents of the University of California
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/backend/storage/file/fd.c,v 1.83 2001/08/04 19:42:34 momjian Exp $
+ * $Header: /cvsroot/pgsql/src/backend/storage/file/fd.c,v 1.84 2001/09/30 18:57:45 tgl Exp $
*
* NOTES:
*
@@ -69,7 +69,7 @@
*
* (Even though most dynamic loaders now use dlopen(3) or the
* equivalent, the OS must still open several files to perform the
- * dynamic loading. Keep this here.)
+ * dynamic loading. And stdin/stdout/stderr count too. Keep this here.)
*/
#ifndef RESERVE_FOR_LD
#define RESERVE_FOR_LD 10
@@ -87,6 +87,14 @@
#define FD_MINFREE 10
#endif
+/*
+ * A number of platforms return values for sysconf(_SC_OPEN_MAX) that are
+ * far beyond what they can really support. This GUC parameter limits what
+ * we will believe.
+ */
+int max_files_per_process = 1000;
+
+
/* Debugging.... */
#ifdef FDDEBUG
@@ -281,29 +289,46 @@ pg_nofile(void)
{
static long no_files = 0;
+ /* need do this calculation only once */
if (no_files == 0)
{
- /* need do this calculation only once */
-#ifndef HAVE_SYSCONF
- no_files = (long) NOFILE;
-#else
+ /*
+ * Ask the system what its files-per-process limit is.
+ */
+#ifdef HAVE_SYSCONF
no_files = sysconf(_SC_OPEN_MAX);
- if (no_files == -1)
+ if (no_files <= 0)
{
-/* tweak for Hurd, which does not support NOFILE */
#ifdef NOFILE
- elog(DEBUG, "pg_nofile: Unable to get _SC_OPEN_MAX using sysconf(); using %d", NOFILE);
no_files = (long) NOFILE;
#else
- elog(FATAL, "pg_nofile: Unable to get _SC_OPEN_MAX using sysconf() and NOFILE is undefined");
+ no_files = (long) max_files_per_process;
#endif
+ elog(DEBUG, "pg_nofile: sysconf(_SC_OPEN_MAX) failed; using %ld",
+ no_files);
}
+#else /* !HAVE_SYSCONF */
+#ifdef NOFILE
+ no_files = (long) NOFILE;
+#else
+ no_files = (long) max_files_per_process;
#endif
+#endif /* HAVE_SYSCONF */
+
+ /*
+ * Some platforms return hopelessly optimistic values. Apply a
+ * configurable upper limit.
+ */
+ if (no_files > (long) max_files_per_process)
+ no_files = (long) max_files_per_process;
+ /*
+ * Make sure we have enough to get by after reserving some for LD.
+ */
if ((no_files - RESERVE_FOR_LD) < FD_MINFREE)
- elog(FATAL, "pg_nofile: insufficient File Descriptors in postmaster to start backend (%ld).\n"
- " O/S allows %ld, Postmaster reserves %d, We need %d (MIN) after that.",
- no_files - RESERVE_FOR_LD, no_files, RESERVE_FOR_LD, FD_MINFREE);
+ elog(FATAL, "pg_nofile: insufficient file descriptors available to start backend.\n"
+ "\tSystem allows %ld, we need at least %d.",
+ no_files, RESERVE_FOR_LD + FD_MINFREE);
no_files -= RESERVE_FOR_LD;
}
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 6efd41c2a8e..c8a9a0afc28 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -4,7 +4,7 @@
* Support for grand unified configuration scheme, including SET
* command, configuration file, and command line options.
*
- * $Header: /cvsroot/pgsql/src/backend/utils/misc/guc.c,v 1.53 2001/09/29 04:02:25 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/backend/utils/misc/guc.c,v 1.54 2001/09/30 18:57:45 tgl Exp $
*
* Copyright 2000 by PostgreSQL Global Development Group
* Written by Peter Eisentraut <peter_e@gmx.net>.
@@ -31,6 +31,7 @@
#include "optimizer/paths.h"
#include "optimizer/planmain.h"
#include "parser/parse_expr.h"
+#include "storage/fd.h"
#include "storage/freespace.h"
#include "storage/lock.h"
#include "storage/proc.h"
@@ -302,6 +303,9 @@ static struct config_int
{"vacuum_mem", PGC_USERSET, &VacuumMem,
8192, 1024, INT_MAX, NULL, NULL},
+ {"max_files_per_process", PGC_BACKEND, &max_files_per_process,
+ 1000, 25, INT_MAX, NULL, NULL},
+
{"debug_level", PGC_USERSET, &DebugLvl,
0, 0, 16, NULL, NULL},
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 789f1a1d08d..fbf555304e3 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -176,10 +176,11 @@
#
#dynamic_library_path = '$libdir'
#australian_timezones = false
-#authentication_timeout = 60 # min 1, max 600
+#authentication_timeout = 60 # min 1, max 600
#deadlock_timeout = 1000
#default_transaction_isolation = 'read committed'
-#max_expr_depth = 10000 # min 10
+#max_expr_depth = 10000 # min 10
+#max_files_per_process = 1000 # min 25
#password_encryption = false
#sql_inheritance = true
#transform_null_equals = false
diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h
index f3e06978a67..3526be1a1ef 100644
--- a/src/include/storage/fd.h
+++ b/src/include/storage/fd.h
@@ -7,7 +7,7 @@
* Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
- * $Id: fd.h,v 1.30 2001/06/11 04:12:29 tgl Exp $
+ * $Id: fd.h,v 1.31 2001/09/30 18:57:45 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -43,6 +43,11 @@ typedef char *FileName;
typedef int File;
+
+/* GUC parameter */
+extern int max_files_per_process;
+
+
/*
* prototypes for functions in fd.c
*/
--
GitLab