Danté's Inferno

Here are the issues we are having:

Tomcat stress tests show that Tomcat reaches max_threads, then hangs, no more connections are made. System must be restarted to allow further processing.

Package Versions:

Apache: Apache HTTP Server Version 2.2 – prefork with mpm
Tomcat: 6.0.20
JK Connector: Same as whatever is bundled in with Apache 2.2 (from customer)
Solaris Solaris 10 10/09 s10s_u8wos_08a SPARC

Current? Startup CommandLine

JAVA_OPTS="-Xss128k -Xms1024m -Xmx2564m -XX:MaxPermSize=512m -XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:SurvivorRatio=6 -XX:NewSize=512M -XX:MaxNewSize=512M -XX:+DisableExplicitGC -Xloggc:gclog_tomcat.txt -XX:+PrintGCTimeStamps -XX:+PrintHeapAtGC -XX:+CMSPermGenSweepingEnabled -XX:+CMSClassUnloadingEnabled"

Apache Version Info

sh-3.00# ./httpd -V
Server version: Apache/2.2.17 (Unix)
Server built:   Nov  4 2010 07:26:31
Server's Module Magic Number: 20051115:25
Server loaded:  APR 1.4.2, APR-Util 1.3.10
Compiled using: APR 1.4.2, APR-Util 1.3.10
Architecture:   32-bit
Server MPM:     Prefork
  threaded:     no
    forked:     yes (variable process count)
Server compiled with....
 -D APACHE_MPM_DIR="server/mpm/prefork"
 -D APR_HAS_SENDFILE
 -D APR_HAS_MMAP
 -D APR_HAVE_IPV6 (IPv4-mapped addresses enabled)
 -D APR_USE_PROC_PTHREAD_SERIALIZE
 -D APR_USE_PTHREAD_SERIALIZE
 -D SINGLE_LISTEN_UNSERIALIZED_ACCEPT
 -D APR_HAS_OTHER_CHILD
 -D AP_HAVE_RELIABLE_PIPED_LOGS
 -D DYNAMIC_MODULE_LIMIT=128
 -D HTTPD_ROOT="/usr/apache2_cgems"
 -D SUEXEC_BIN="/usr/apache2_cgems/bin/suexec"
 -D DEFAULT_PIDLOG="logs/httpd.pid"
 -D DEFAULT_SCOREBOARD="logs/apache_runtime_status"
 -D DEFAULT_LOCKFILE="logs/accept.lock"
 -D DEFAULT_ERRORLOG="logs/error_log"
 -D AP_TYPES_CONFIG_FILE="conf/mime.types"
 -D SERVER_CONFIG_FILE="conf/httpd.conf"

prstat output when hung

   PID USERNAME USR SYS TRP TFL DFL LCK SLP LAT VCX ICX SCL SIG PROCESS/LWPID

 13030 ems       43  20 0.2 0.0 0.1 6.8  29 0.0  15   6  1K   0 java/349
 13030 ems      3.3 0.5 0.1 0.0 0.1 0.5  95 0.0   4   0 328   0 java/364
 13030 ems      2.9 0.5 0.0 0.0 0.0  62  34 0.0  10   0 353   0 java/344
 13030 ems      2.8 0.5 0.0 0.0 0.0 0.3  96 0.0   4   0 316   0 java/377
 13030 ems      2.2 0.1 0.0 0.0 0.0  28  70 0.0   4   0  19   0 java/10194
 13030 ems      2.0 0.1 0.0 0.0 0.0  34  64 0.0   4   0  20   0 java/10132
 13030 ems      2.0 0.1 0.0 0.0 0.0  49  49 0.0   3   0  16   0 java/10166
 13030 ems      2.0 0.1 0.0 0.0 0.0  38  60 0.0   3   0  16   0 java/10123
 13030 ems      1.9 0.1 0.0 0.0 0.0  36  62 0.0   3   0  18   0 java/10157
 13030 ems      1.9 0.1 0.0 0.0 0.0  38  60 0.0   3   0  17   0 java/10143
 13030 ems      1.8 0.1 0.0 0.0 0.0  39  59 0.0   3   0  17   0 java/10126
 13030 ems      1.8 0.1 0.0 0.0 0.0  35  63 0.0   3   0  18   0 java/10174
 13030 ems      1.8 0.1 0.0 0.0 0.0  33  65 0.0   3   0  17   0 java/10133
 13030 ems      1.8 0.1 0.0 0.0 0.0  36  62 0.0   3   0  17   0 java/10145
 13030 ems      1.8 0.1 0.0 0.0 0.0  39  59 0.0   3   0  17   0 java/10162
 13030 ems      1.7 0.1 0.0 0.0 0.0  50  48 0.0   3   0  16   0 java/10208
 13030 ems      1.7 0.1 0.0 0.0 0.0  43  55 0.0   3   0  16   0 java/10211
 13030 ems      1.7 0.1 0.0 0.0 0.0  47  51 0.0   3   0  15   0 java/10137
 13030 ems      1.6 0.1 0.0 0.0 0.0  35  63 0.0   3   0  16   0 java/10152
 13030 ems      1.6 0.1 0.0 0.0 0.0  34  65 0.0   3   0  16   0 java/10067
 13030 ems      1.6 0.1 0.0 0.0 0.0  48  50 0.0   3   0  15   0 java/10191
 13030 ems      1.6 0.1 0.0 0.0 0.0  40  59 0.0   3   0  15   0 java/10148
 13030 ems      1.5 0.0 0.0 0.0 0.0  37  62 0.0   3   0  15   0 java/10173
 13030 ems      1.5 0.1 0.0 0.0 0.0  33  66 0.0   3   0  15   0 java/9973
 13030 ems      1.5 0.0 0.0 0.0 0.0  44  54 0.0   3   0  14   0 java/10165
 13030 ems      1.5 0.0 0.0 0.0 0.0  53  45 0.0   2   0  13   0 java/10150
 13030 ems      1.4 0.0 0.0 0.0 0.0  40  58 0.0   3   0  14   0 java/10076
 13030 ems      1.4 0.1 0.0 0.0 0.0  41  58 0.0   3   0  14   0 java/10006
 13030 ems      1.3 0.0 0.0 0.0 0.0  67  31 0.0   2   0  12   0 java/10207
 13030 ems      1.3 0.0 0.0 0.0 0.0  38  60 0.0   3   0  14   0 java/9924
 13030 ems      1.3 0.0 0.0 0.0 0.0  41  58 0.0   2   0  13   0 java/10080
 13030 ems      1.3 0.0 0.0 0.0 0.0  45  54 0.0   3   0  13   0 java/9991
 13030 ems      1.2 0.1 0.0 0.0 0.0  40  59 0.0   3   0  13   0 java/9869
 13030 ems      1.2 0.0 0.0 0.0 0.0  42  56 0.0   2   0  13   0 java/10037
 13030 ems      1.2 0.0 0.0 0.0 0.0  60  39 0.0   2   0  11   0 java/10210
 13030 ems      1.2 0.0 0.0 0.0 0.0  50  48 0.0   2   0  12   0 java/10113
 13030 ems      1.2 0.1 0.0 0.0 0.0  45  54 0.0   3   0  12   0 java/9964
 13030 ems      1.2 0.0 0.0 0.0 0.0  57  42 0.0   2   0  10   0 java/10206
 13030 ems      0.8 0.4 0.1 0.0 0.0  98 0.1 0.0  27   0  29   0 java/108
Total: 1 processes, 828 lwps, load averages: 17.17, 22.55, 22.69

workers.properties

# Define 1 real worker using ajp13
worker.list=worker1,worker2,worker3,worker4
worker.maintain=10

# Set properties for worker1 (ajp13)
worker.worker1.type=ajp13
worker.worker1.host=localhost
worker.worker1.port=8019
worker.worker1.lbfactor=1
worker.worker1.connection_pool_size=1
worker.worker1.connection_pool_timeout=10
worker.worker1.socket_keepalive=1
worker.worker1.socket_timeout=300
worker.worker1.cache_timeout=10

# Set properties for worker2 (ajp13)
worker.worker2.type=ajp13
worker.worker2.host=localhost
worker.worker2.port=8019
worker.worker2.lbfactor=1
worker.worker2.connection_pool_size=1
worker.worker2.connection_pool_timeout=10
worker.worker2.socket_keepalive=1
worker.worker2.socket_timeout=300
worker.worker2.cache_timeout=10

# Set properties for worker3 (ajp13)
worker.worker3.type=ajp13
worker.worker3.host=localhost
worker.worker3.port=8019
worker.worker3.lbfactor=1
worker.worker3.connection_pool_size=1
worker.worker3.connection_pool_timeout=10
worker.worker3.socket_keepalive=1
worker.worker3.socket_timeout=300
worker.worker3.cache_timeout=10

# Set properties for worker4 (ajp13)
worker.worker4.type=ajp13
worker.worker4.host=localhost
worker.worker4.port=8019
worker.worker4.lbfactor=1
worker.worker4.connection_pool_size=1
worker.worker4.connection_pool_timeout=10
worker.worker4.socket_keepalive=1
worker.worker4.socket_timeout=300
worker.worker4.cache_timeout=10

httpd.conf

#
# This is the main Apache HTTP server configuration file.  It contains the
# configuration directives that give the server its instructions.
# See <URL:http://httpd.apache.org/docs/2.2> for detailed information.
# In particular, see
# <URL:http://httpd.apache.org/docs/2.2/mod/directives.html>
# for a discussion of each configuration directive.
#
# Do NOT simply read the instructions in here without understanding
# what they do.  They're here only as hints or reminders.  If you are unsure
# consult the online docs. You have been warned.
#
# Configuration and logfile names: If the filenames you specify for many
# of the server's control files begin with "/" (or "drive:/" for Win32), the
# server will use that explicit path.  If the filenames do *not* begin
# with "/", the value of ServerRoot is prepended -- so "logs/foo_log"
# with ServerRoot set to "/usr/apache2_cgems" will be interpreted by the
# server as "/usr/apache2_cgems/logs/foo_log".

#
# ServerRoot: The top of the directory tree under which the server's
# configuration, error, and log files are kept.
#
# Do not add a slash at the end of the directory path.  If you point
# ServerRoot at a non-local disk, be sure to point the LockFile directive
# at a local disk.  If you wish to share the same ServerRoot for multiple
# httpd daemons, you will need to change at least LockFile and PidFile.
#
ServerRoot "/usr/apache2_cgems"

#
# Listen: Allows you to bind Apache to specific IP addresses and/or
# ports, instead of the default. See also the <VirtualHost>
# directive.
#
# Change this to Listen on specific IP addresses as shown below to
# prevent Apache from glomming onto all bound IP addresses.
#
#Listen 12.34.56.78:80
Listen 80

#
# Dynamic Shared Object (DSO) Support
#
# To be able to use the functionality of a module which was built as a DSO you
# have to place corresponding `LoadModule' lines at this location so the
# directives contained in it are actually available _before_ they are used.
# Statically compiled modules (those listed by `httpd -l') do not need
# to be loaded here.
#
# Example:
# LoadModule foo_module modules/mod_foo.so
#
LoadModule authn_file_module libexec/mod_authn_file.so
LoadModule authn_dbm_module libexec/mod_authn_dbm.so
LoadModule authn_anon_module libexec/mod_authn_anon.so
LoadModule authn_dbd_module libexec/mod_authn_dbd.so
LoadModule authn_default_module libexec/mod_authn_default.so
LoadModule authz_host_module libexec/mod_authz_host.so
LoadModule authz_groupfile_module libexec/mod_authz_groupfile.so
LoadModule authz_user_module libexec/mod_authz_user.so
LoadModule authz_dbm_module libexec/mod_authz_dbm.so
LoadModule authz_owner_module libexec/mod_authz_owner.so
LoadModule authz_default_module libexec/mod_authz_default.so
LoadModule auth_basic_module libexec/mod_auth_basic.so
LoadModule auth_digest_module libexec/mod_auth_digest.so
LoadModule dbd_module libexec/mod_dbd.so
LoadModule dumpio_module libexec/mod_dumpio.so
LoadModule reqtimeout_module libexec/mod_reqtimeout.so
LoadModule ext_filter_module libexec/mod_ext_filter.so
LoadModule include_module libexec/mod_include.so
LoadModule filter_module libexec/mod_filter.so
LoadModule substitute_module libexec/mod_substitute.so
LoadModule deflate_module libexec/mod_deflate.so
LoadModule log_config_module libexec/mod_log_config.so
LoadModule log_forensic_module libexec/mod_log_forensic.so
LoadModule logio_module libexec/mod_logio.so
LoadModule env_module libexec/mod_env.so
LoadModule mime_magic_module libexec/mod_mime_magic.so
LoadModule cern_meta_module libexec/mod_cern_meta.so
LoadModule expires_module libexec/mod_expires.so
LoadModule headers_module libexec/mod_headers.so
LoadModule ident_module libexec/mod_ident.so
LoadModule usertrack_module libexec/mod_usertrack.so
LoadModule unique_id_module libexec/mod_unique_id.so
LoadModule setenvif_module libexec/mod_setenvif.so
LoadModule version_module libexec/mod_version.so
LoadModule proxy_module libexec/mod_proxy.so
LoadModule proxy_connect_module libexec/mod_proxy_connect.so
LoadModule proxy_ftp_module libexec/mod_proxy_ftp.so
LoadModule proxy_http_module libexec/mod_proxy_http.so
LoadModule proxy_scgi_module libexec/mod_proxy_scgi.so
LoadModule proxy_ajp_module libexec/mod_proxy_ajp.so
LoadModule proxy_balancer_module libexec/mod_proxy_balancer.so
#LoadModule ssl_module libexec/mod_ssl.so
LoadModule mime_module libexec/mod_mime.so
LoadModule dav_module libexec/mod_dav.so
LoadModule status_module libexec/mod_status.so
LoadModule autoindex_module libexec/mod_autoindex.so
LoadModule asis_module libexec/mod_asis.so
LoadModule info_module libexec/mod_info.so
LoadModule suexec_module libexec/mod_suexec.so
LoadModule cgi_module libexec/mod_cgi.so
LoadModule dav_fs_module libexec/mod_dav_fs.so
LoadModule vhost_alias_module libexec/mod_vhost_alias.so
LoadModule negotiation_module libexec/mod_negotiation.so
LoadModule dir_module libexec/mod_dir.so
LoadModule imagemap_module libexec/mod_imagemap.so
LoadModule actions_module libexec/mod_actions.so
LoadModule speling_module libexec/mod_speling.so
LoadModule userdir_module libexec/mod_userdir.so
LoadModule alias_module libexec/mod_alias.so
LoadModule rewrite_module libexec/mod_rewrite.so

<IfModule !mpm_netware_module>
<IfModule !mpm_winnt_module>
#
# If you wish httpd to run as a different user or group, you must run
# httpd as root initially and it will switch.
#
# User/Group: The name (or #number) of the user/group to run httpd as.
# It is usually good practice to create a dedicated user and group for
# running httpd, as with most system services.
#
User ems
Group ems
</IfModule>
</IfModule>

# 'Main' server configuration
#
# The directives in this section set up the values used by the 'main'
# server, which responds to any requests that aren't handled by a
# <VirtualHost> definition.  These values also provide defaults for
# any <VirtualHost> containers you may define later in the file.
#
# All of these directives may appear inside <VirtualHost> containers,
# in which case these default settings will be overridden for the
# virtual host being defined.
#

#
# ServerAdmin: Your address, where problems with the server should be
# e-mailed.  This address appears on some server-generated pages, such
# as error documents.  e.g. admin@your-domain.com
#
ServerAdmin you@example.com

#
# ServerName gives the name and port that the server uses to identify itself.
# This can often be determined automatically, but we recommend you specify
# it explicitly to prevent problems during startup.
#
# If your host doesn't have a registered DNS name, enter its IP address here.
#
#ServerName www.example.com:80
#
ServerName mycompany.com:80

#
# DocumentRoot: The directory out of which you will serve your
# documents. By default, all requests are taken from this directory, but
# symbolic links and aliases may be used to point to other locations.
#
DocumentRoot "/usr/apache2_cgems/htdocs"

#
# Each directory to which Apache has access can be configured with respect
# to which services and features are allowed and/or disabled in that
# directory (and its subdirectories).
#
# First, we configure the "default" to be a very restrictive set of
# features.
#
<Directory />
    Options FollowSymLinks
    AllowOverride None
</Directory>

#
# Note that from this point forward you must specifically allow
# particular features to be enabled - so if something's not working as
# you might expect, make sure that you have specifically enabled it
# below.
#

#
# This should be changed to whatever you set DocumentRoot to.
#
<Directory "/usr/apache2_cgems/htdocs">
#
# Possible values for the Options directive are "None", "All",
# or any combination of:
#   Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews
#
# Note that "MultiViews" must be named *explicitly* --- "Options All"
# doesn't give it to you.
#
# The Options directive is both complicated and important.  Please see
    # http://httpd.apache.org/docs/2.2/mod/core.html#options
# for more information.
#
    Options Indexes FollowSymLinks

#
# AllowOverride controls what directives may be placed in .htaccess files.
# It can be "All", "None", or any combination of the keywords:
    #   Options FileInfo AuthConfig Limit
#
    AllowOverride None

#
# Controls who can get stuff from this server.
#
    Order allow,deny
    Allow from all

</Directory>

#
# DirectoryIndex: sets the file that Apache will serve if a directory
# is requested.
#
<IfModule dir_module>
    DirectoryIndex index.html
</IfModule>

#
# The following lines prevent .htaccess and .htpasswd files from being
# viewed by Web clients.
#
<FilesMatch "^\.ht">
    Order allow,deny
    Deny from all
</FilesMatch>

#
# ErrorLog: The location of the error log file.
# If you do not specify an ErrorLog directive within a <VirtualHost>
# container, error messages relating to that virtual host will be
# logged here.  If you *do* define an error logfile for a <VirtualHost>
# container, that host's errors will be logged there and not here.
#
ErrorLog "logs/error_log"

#
# LogLevel: Control the number of messages logged to the error_log.
# Possible values include: debug, info, notice, warn, error, crit,
# alert, emerg.
#
LogLevel warn

<IfModule log_config_module>
    #
    # The following directives define some format nicknames for use with
    # a CustomLog directive (see below).
    #
    LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
    LogFormat "%h %l %u %t \"%r\" %>s %b" common
    LogFormat "%{Referer}i -> %U" referer
	LogFormat "%{User-agent}i" agent

    <IfModule logio_module>
      # You need to enable mod_logio.c to use %I and %O
      #LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
    </IfModule>

    #
    # The location and format of the access logfile (Common Logfile Format).
    # If you do not define any access logfiles within a <VirtualHost>
    # container, they will be logged here.  Contrariwise, if you *do*
    # define per-<VirtualHost> access logfiles, transactions will be
    # logged therein and *not* in this file.
    #
    CustomLog "logs/access_log" common

    #
    # If you prefer a logfile with access, agent, and referer information
    # (Combined Logfile Format) you can use the following directive.
    #
    #CustomLog "logs/access_log" combined
</IfModule>

<IfModule alias_module>
    #
    # Redirect: Allows you to tell clients about documents that used to
    # exist in your server's namespace, but do not anymore. The client
    # will make a new request for the document at its new location.
    # Example:
    # Redirect permanent /foo http://www.example.com/bar

    #
    # Alias: Maps web paths into filesystem paths and is used to
    # access content that does not live under the DocumentRoot.
    # Example:
    # Alias /webpath /full/filesystem/path
    #
    # If you include a trailing / on /webpath then the server will
    # require it to be present in the URL.  You will also likely
    # need to provide a <Directory> section to allow access to
    # the filesystem path.

    #
    # ScriptAlias: This controls which directories contain server scripts.
    # ScriptAliases are essentially the same as Aliases, except that
    # documents in the target directory are treated as applications and
    # run by the server when requested rather than as documents sent to the
    # client.  The same rules about trailing "/" apply to ScriptAlias
    # directives as to Alias.
    #
    ScriptAlias /cgi-bin/ "/usr/apache2_cgems/cgi-bin/"

</IfModule>

<IfModule cgid_module>
    #
    # ScriptSock: On threaded servers, designate the path to the UNIX
    # socket used to communicate with the CGI daemon of mod_cgid.
    #
    #Scriptsock logs/cgisock
</IfModule>

#
# "/usr/apache2_cgems/cgi-bin" should be changed to whatever your ScriptAliased
# CGI directory exists, if you have that configured.
#
<Directory "/usr/apache2_cgems/cgi-bin">
    AllowOverride None
    Options None
    Order allow,deny
    Allow from all
</Directory>

#
# DefaultType: the default MIME type the server will use for a document
# if it cannot otherwise determine one, such as from filename extensions.
# If your server contains mostly text or HTML documents, "text/plain" is
# a good value.  If most of your content is binary, such as applications
# or images, you may want to use "application/octet-stream" instead to
# keep browsers from trying to display binary files as though they are
# text.
#
DefaultType text/plain

<IfModule mime_module>
    #
    # TypesConfig points to the file containing the list of mappings from
    # filename extension to MIME-type.
    #
    TypesConfig conf/mime.types

    #
    # AddType allows you to add to or override the MIME configuration
    # file specified in TypesConfig for specific file types.
#
    #AddType application/x-gzip .tgz
#
# AddEncoding allows you to have certain browsers uncompress
# information on the fly. Note: Not all browsers support this.
#
#AddEncoding x-compress .Z
#AddEncoding x-gzip .gz .tgz
#
# If the AddEncoding directives above are commented-out, then you
# probably should define those extensions to indicate media types:
#
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz

#
# AddHandler allows you to map certain file extensions to "handlers":
# actions unrelated to filetype. These can be either built into the server
# or added with the Action directive (see below)
#
# To use CGI scripts outside of ScriptAliased directories:
# (You will also need to add "ExecCGI" to the "Options" directive.)
#
#AddHandler cgi-script .cgi

# For type maps (negotiated resources):
AddHandler type-map var

#
# Filters allow you to process content before it is sent to the client.
#
# To parse .shtml files for server-side includes (SSI):
# (You will also need to add "Includes" to the "Options" directive.)
#
#AddType text/html .shtml
#AddOutputFilter INCLUDES .shtml
</IfModule>

#
# The mod_mime_magic module allows the server to use various hints from the
# contents of the file itself to determine its type.  The MIMEMagicFile
# directive tells the module where the hint definitions are located.
#
MIMEMagicFile conf/magic

#
# Customizable error responses come in three flavors:
# 1) plain text 2) local redirects 3) external redirects
#
# Some examples:
#ErrorDocument 500 "The server made a boo boo."
#ErrorDocument 404 /missing.html
#ErrorDocument 404 "/cgi-bin/missing_handler.pl"
#ErrorDocument 402 http://www.example.com/subscription_info.html
#

#
# EnableMMAP and EnableSendfile: On systems that support it,
# memory-mapping or the sendfile syscall is used to deliver
# files.  This usually improves server performance, but must
# be turned off when serving from networked-mounted
# filesystems or if support for these functions is otherwise
# broken on your system.
#
#EnableMMAP off
#EnableSendfile off

# Supplemental configuration
#
# The configuration files in the conf/extra/ directory can be
# included to add extra features or to modify the default configuration of
# the server, or you may simply copy their contents here and change as
# necessary.

# Server-pool management (MPM specific)
Include conf/extra/httpd-mpm.conf

# Multi-language error messages
Include conf/extra/httpd-multilang-errordoc.conf

# Fancy directory listings
Include conf/extra/httpd-autoindex.conf

# Language settings
Include conf/extra/httpd-languages.conf

# User home directories
#Include conf/extra/httpd-userdir.conf

# Real-time info on requests and configuration
#Include conf/extra/httpd-info.conf

# Virtual hosts
#Include conf/extra/httpd-vhosts.conf

# Local access to the Apache HTTP Server Manual
Include conf/extra/httpd-manual.conf

# Distributed authoring and versioning (WebDAV)
Include conf/extra/httpd-dav.conf

# Various default settings
Include conf/extra/httpd-default.conf

# Secure (SSL/TLS) connections
#Include conf/extra/httpd-ssl.conf
#
# Note: The following must must be present to support
#       starting without SSL on platforms with no /dev/random equivalent
#       but a statically compiled-in mod_ssl.
#
<IfModule ssl_module>
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
</IfModule>
### BEGIN SCCAN_NMS_CONFIG mycompany SCCaN NMS Configuration
###
### AUTOMATICALLY GENERATED CONFIGURATION, DO NOT MODIFY THIS SECTION.
#Alias /OvDocs/ "C:/Program Files/HP OpenView/www/htdocs/"
#Alias /OvBackgrounds/ "C:/Program Files/HP OpenView/backgrounds/"
#ScriptAlias  /OpenView    "C:/Program Files/HP OpenView/www/cgi-bin/OpenView"
#ScriptAlias /OvCgi/ "C:/Program Files/HP OpenView/www/cgi-bin/"
#ScriptAlias  /ovcgi    "C:/Program Files/HP OpenView/www/cgi-bin"

#Added for HTTPS upload support
ScriptAlias /cgi-bin-putusers "/usr/apache2_cgems/cgi-bin-putusers"

<Directory "/mycompany/apps/myfm/fmserver/webapps" >
        Script PUT /cgi-bin-putusers/put.cgi
</Directory>

<Directory "/usr/apache2_cgems/cgi-bin-putusers">
    AllowOverride None
    <Limit GET POST PUT>
      Allow from all
    </Limit>
    <LimitExcept GET POST PUT>
      Deny from all
    </LimitExcept>
</Directory>

#LoadModule auth_digest_module libexec/mod_auth_digest.so
#LoadModule proxy_module libexec/mod_proxy.so
#LoadModule proxy_http_module libexec/mod_proxy_http.so
#LoadModule rewrite_module libexec/mod_rewrite.so

#Added for remote server execution.
#wsm_hostname should be changed to the hostname of the WSM.
#<IfModule mod_proxy.c>
#    ProxyPass /prov/online http://[WSM_FQDN]:4083/prov/online
#    ProxyPass /prov http://[WSM_FQDN]:4082/prov
#</IfModule>

#Added for virtual hosts
include /usr/apache2_cgems/conf/vhost.conf

# Load auth_external_module module
#LoadModule auth_external_module libexec/mod_auth_external.so
LoadModule authnz_external_module libexec/mod_authnz_external.so

# Load mod_jk module
# mod_jk is in the modules directory
LoadModule jk_module libexec/mod_jk.so 

# Declare the module for <IfModule directive>
<IfModule mod_jk.c>

# Where to find workers.properties
# Put workers.properties in the Tomcat directory
JkWorkersFile "/mycompany/apps/myfm/fmserver/Tomcat/conf/workers.properties"

# Where to put jk logs
# Apache logs (put mod_jk.log next to access_log)
#JkLogFile /usr/apache2_cgems/logs/mod_jk.log
#Adding code for piping mod_jk log is piped
#once in a month i.e every 30 days. The following is the syntax used
#for piping the logs. If instead of the time if size needs to be used then
#replace the time with the size in MB
JkLogFile /usr/apache2_cgems/logs/mod_jk.log 

# Set the jk log level [debug/error/info]
JkLogLevel error 

# Select the log format
JkLogStampFormat "[%a %b %d %H:%M:%S %Y] " 

# JkOptions indicate to send SSL KEY SIZE,
JkOptions +ForwardKeySize +ForwardURICompat -ForwardDirectories 

# JkRequestLogFormat set the request format
JkRequestLogFormat "%w %V %T" 

# Send everything for context /MyCfg/servlet/ to worker named worker1 (ajp13)
#JkMount /MyCfg/servlet/* worker1
#JkMount /MyCfg/* worker1

#JkMount /MyCfg/servlet/Login worker1
#JkMount /MyCfg/servlet/mycompanyWiMax worker1
#JkMount /MyCfg/servlet/*.jnlp worker1

#JkMount /MyCfg/*.jnlp worker1
#JkMount /MyCfg/servlet/*AsyncMsgPollServlet worker4
JkMount  /ACT worker2
JkMount  /ACT/* worker2

# Send the Online Help requests to worker3
#JkMount /MyoEMHelp/* worker3

# Send Prov Client requests to worker named worker2 (ajp13)
#JKMount /provClient worker2
#JKMount /provClient/* worker2
#JKMount /wonClient worker2
#JKMount /wonClient/* worker2

#Authenticate NE Users
<Location /MyCfg/servlet/NeConfig.jnlp>
#AuthType Digest
AuthName "mycompany EMS"
#AuthDigestFile "/usr/apache2_cgems/passwd/digest"
AuthType Basic
AuthExternal pwauth
AuthBasicProvider external
AuthBasicAuthoritative off
Require valid-user
</Location>

#Authenticate EM Users
<Location /MyCfg/servlet/EmConfig.jnlp>
#AuthType Digest
AuthName "mycompany EMS"
#AuthDigestFile "/usr/apache2_cgems/passwd/digest"
AuthType Basic
AuthExternal pwauth
AuthBasicProvider external
AuthBasicAuthoritative off
Require valid-user
</Location>

#Authenticate Account Manager Users
<Location /mycompany/AccountManager.htm>
#AuthType Digest
AuthName "mycompany EMS"
#AuthDigestFile "/usr/apache2_cgems/passwd/digest"
AuthType Basic
AuthExternal pwauth
AuthBasicProvider external
AuthBasicAuthoritative off
Require valid-user
</Location>

#Authenticate NBI Users
<Location /MyCfg/services/*>
#AuthType Digest
AuthName "mycompany NBI"
#AuthDigestFile "/usr/apache2_cgems/passwd/digest"
AuthType Basic
AuthExternal pwauth
AuthBasicProvider external
AuthBasicAuthoritative off
Require valid-user
</Location>

#Authenticate NBI Users for fileupload
<Location /MyCfg/filetransfer/fileupload/*>
#AuthType Digest
AuthName "mycompany NBI"
#AuthDigestFile "/usr/apache2_cgems/passwd/digest"
AuthType Basic
AuthExternal pwauth
AuthBasicProvider external
AuthBasicAuthoritative off
Require valid-user
</Location>

#Authenticate NBI Users for filedownload
<Location /MyCfg/filetransfer/filedownload/*>
#AuthType Digest
AuthName "mycompany NBI"
#AuthDigestFile "/usr/apache2_cgems/passwd/digest"
AuthType Basic
AuthExternal pwauth
AuthBasicProvider external
AuthBasicAuthoritative off
Require valid-user
</Location>

<Location /MyCfg/services2/*>
#AuthType Digest
AuthName "mycompany NBI"
#AuthDigestFile "/usr/apache2_cgems/passwd/digest"
AuthType Basic
AuthExternal pwauth
AuthBasicProvider external
AuthBasicAuthoritative off
Require valid-user
</Location>

#Authenticate NMS Viewer
<Location /MyCfg/servlet/Login>
Allow from all
#AuthType Digest
#AuthName "mycompany EMS"
#AuthDigestFile "/usr/apache2_cgems/passwd/digest"
#AuthType Basic
#AuthExternal pwauth
#Require valid-user
AuthBasicProvider external
AuthBasicAuthoritative off
</Location>

</IfModule>

# For STIG
# enble SSLv3 and TLSv1, but not SSLv2
<IfModule mod_ssl.c>
SSLProtocol all -SSLv2
SSLCipherSuite HIGH:MEDIUM:+EXP:+eNULL
</IfModule>

# Disable Trace and Track DEBUG functions in apache
<IfModule mod_rewrite.c>
RewriteEngine On
#RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK|HEAD)
RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK)
RewriteRule .* - [F]
</IfModule>
TraceEnable off
### END SCCAN_NMS_CONFIG mycompany SCCaN NMS Configuration

# For STIG
# enble SSLv3 and TLSv1, but not SSLv2
<IfModule mod_ssl.c>
SSLProtocol all -SSLv2
SSLCipherSuite HIGH:MEDIUM:+EXP:+eNULL
</IfModule>

TraceEnable off

server.xml

<?xml version='1.0' encoding='utf-8'?>
<!--
  Licensed to the Apache Software Foundation (ASF) under one or more
  contributor license agreements.  See the NOTICE file distributed with
  this work for additional information regarding copyright ownership.
  The ASF licenses this file to You under the Apache License, Version 2.0
  (the "License"); you may not use this file except in compliance with
  the License.  You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
-->
<!-- Note:  A "Server" is not itself a "Container", so you may not
     define subcomponents such as "Valves" at this level.
     Documentation at /docs/config/server.html
 -->
<Server port="8005" shutdown="SHUTDOWN">

  <!--APR library loader. Documentation at /docs/apr.html -->
  <Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />
  <!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
  <Listener className="org.apache.catalina.core.JasperListener" />
  <!-- JMX Support for the Tomcat server. Documentation at /docs/non-existent.html -->
  <Listener className="org.apache.catalina.mbeans.ServerLifecycleListener" />
  <Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />

  <!-- Global JNDI resources
       Documentation at /docs/jndi-resources-howto.html
  -->
  <GlobalNamingResources>
    <!-- Editable user database that can also be used by
         UserDatabaseRealm to authenticate users
    -->
    <Resource name="UserDatabase" auth="Container"
              type="org.apache.catalina.UserDatabase"
              description="User database that can be updated and saved"
              factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
              pathname="conf/tomcat-users.xml" />
  </GlobalNamingResources>

  <!-- A "Service" is a collection of one or more "Connectors" that share
       a single "Container" Note:  A "Service" is not itself a "Container",
       so you may not define subcomponents such as "Valves" at this level.
       Documentation at /docs/config/service.html
   -->
  <Service name="Catalina">

    <!--The connectors can use a shared executor, you can define one or more named thread pools-->
    <!--
    <Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
        maxThreads="150" minSpareThreads="4"/>
    -->

    <!-- A "Connector" represents an endpoint by which requests are received
         and responses are returned. Documentation at :
         Java HTTP Connector: /docs/config/http.html (blocking & non-blocking)
         Java AJP  Connector: /docs/config/ajp.html
         APR (HTTP/AJP) Connector: /docs/apr.html
         Define a non-SSL HTTP/1.1 Connector on port 8080
    -->
    <Connector port="8110" protocol="HTTP/1.1"
               connectionTimeout="10000"
               redirectPort="8445" />
    <!-- A "Connector" using the shared thread pool-->
    <!--
    <Connector executor="tomcatThreadPool"
               port="8080" protocol="HTTP/1.1"
               connectionTimeout="20000"
               redirectPort="8445" />
    -->
    <!-- Define a SSL HTTP/1.1 Connector on port 8445
         This connector uses the JSSE configuration, when using APR, the
         connector should be using the OpenSSL style configuration
         described in the APR documentation -->
    <Connector port="8445" protocol="HTTP/1.1" SSLEnabled="true"
               maxThreads="150" scheme="https" secure="true"
               clientAuth="false" sslProtocol="TLS" keystoreFile="/motorola/apps/mofm/saml/SAML/conf/keystore.jks" keystorePass="changeit" />

    <!-- Define an AJP 1.3 Connector on port 8009 -->
    <Connector port="8019" connectionTimeout="10000" maxThreads="750" minSpareThreads="20" maxSpareThreads="50" request.TomcatAuthentication="false" protocol="AJP/1.3" redirectPort="8445" />

    <!-- An Engine represents the entry point (within Catalina) that processes
         every request.  The Engine implementation for Tomcat stand alone
         analyzes the HTTP headers included with the request, and passes them
         on to the appropriate Host (virtual host).
         Documentation at /docs/config/engine.html -->

    <!-- You should set jvmRoute to support load-balancing via AJP ie :
    <Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
    -->
    <Engine name="Catalina" defaultHost="localhost">

      <!--For clustering, please take a look at documentation at:
          /docs/cluster-howto.html  (simple how to)
          /docs/config/cluster.html (reference documentation) -->
      <!--
      <Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
      -->        

      <!-- The request dumper valve dumps useful debugging information about
           the request and response data received and sent by Tomcat.
           Documentation at: /docs/config/valve.html -->
      <!--
      <Valve className="org.apache.catalina.valves.RequestDumperValve"/>
      -->

      <!-- This Realm uses the UserDatabase configured in the global JNDI
           resources under the key "UserDatabase".  Any edits
           that are performed against this UserDatabase are immediately
           available for use by the Realm.  -->
      <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
             resourceName="UserDatabase"/>

      <!-- Define the default virtual host
           Note: XML Schema validation will not work with Xerces 2.2.
       -->
<Host name="localhost"  appBase="/motorola/apps/mofm/fmserver/webapps"
            unpackWARs="true" autoDeploy="true"
            xmlValidation="false" xmlNamespaceAware="false">

        <!-- SingleSignOn valve, share authentication between web applications
             Documentation at: /docs/config/valve.html -->
        <!--
        <Valve className="org.apache.catalina.authenticator.SingleSignOn" />
        -->

        <!-- Access log processes all example.
             Documentation at: /docs/config/valve.html -->
        <!--
        <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
               prefix="localhost_access_log." suffix=".txt" pattern="common" resolveHosts="false"/>
        -->

      </Host>
    </Engine>
  </Service>
</Server>

Log File

tomcat_stack_dump.tar.gz

Please leave comments or e-mails if you have any ideas.