diff -u -r -N squid-3.1.3/ChangeLog squid-3.1.4/ChangeLog --- squid-3.1.3/ChangeLog 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/ChangeLog 2010-05-31 01:21:11.000000000 +1200 @@ -1,3 +1,24 @@ +Changes to squid-3.1.4 (30 May 2010): + + - Bug 2933: Verification of the max. port number for WCCP2 dynamic service + - Bug 2924: RADIUS helper compile issues + - Bug 2922: Fix assertion failed: HttpHeader.cc: "Headers[id].stat.aliveCount" + - Bug 2919: tcp_outgoing_address ACLs not obeying acl_uses_indirect_client + - Bug 2896: Fix assertion failed: comm.cc:2063: "!fd_table[fd].closing()" + - Bug 2879: pt2: 3.0 regression in headers end finding + - Bug 2877: pt2: only output zero-size warning on reverse-proxy requests + - Bug 2876: FD_SETSIZE override not working on all linux distributions + - Bug 2810: common log format generates 2 lines of syslog + - Bug 2789: Optimize unlimited memory pools, and correctly handle limits over 2GB + - Bug 2753: Fall back on IPv4 if IPv6 is not present + - Bug 2697: Adaptation leaks and extra requests after reconfiguration + - Bug 2633: Fix Ecap::HeaderRep::value(name) fails when there is no named header field + - Change LDAP helpers to default to LDAP version 3 if available + - Add Joomla and Salted Hash support to squid_db_auth helper + - Fixed IpAddress port printing for ports higher than 9999 + - Disable chunked memory pooling by default. + - ... and several build errors. + Changes to squid-3.1.3 (02 May 2010): - Remove: Advertise 1.1 on replies diff -u -r -N squid-3.1.3/compat/fdsetsize.h squid-3.1.4/compat/fdsetsize.h --- squid-3.1.3/compat/fdsetsize.h 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/compat/fdsetsize.h 2010-05-31 01:21:12.000000000 +1200 @@ -40,6 +40,7 @@ #if (__GLIBC__ > 2) || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 2) #if SQUID_MAXFD > DEFAULT_FD_SETSIZE #include +#include #undef __FD_SETSIZE #define __FD_SETSIZE SQUID_MAXFD #endif diff -u -r -N squid-3.1.3/configure squid-3.1.4/configure --- squid-3.1.3/configure 2010-05-02 22:48:00.000000000 +1200 +++ squid-3.1.4/configure 2010-05-31 01:21:49.000000000 +1200 @@ -1,7 +1,7 @@ #! /bin/sh # From configure.in Revision. # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for Squid Web Proxy 3.1.3. +# Generated by GNU Autoconf 2.62 for Squid Web Proxy 3.1.4. # # Report bugs to . # @@ -751,8 +751,8 @@ # Identity of this package. PACKAGE_NAME='Squid Web Proxy' PACKAGE_TARNAME='squid' -PACKAGE_VERSION='3.1.3' -PACKAGE_STRING='Squid Web Proxy 3.1.3' +PACKAGE_VERSION='3.1.4' +PACKAGE_STRING='Squid Web Proxy 3.1.4' PACKAGE_BUGREPORT='http://www.squid-cache.org/bugs/' ac_unique_file="src/main.cc" @@ -1104,7 +1104,6 @@ enable_ntlm_fail_open enable_external_acl_helpers with_valgrind_debug -enable_mempools enable_win32_service enable_unlinkd enable_stacktraces @@ -1692,7 +1691,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Squid Web Proxy 3.1.3 to adapt to many kinds of systems. +\`configure' configures Squid Web Proxy 3.1.4 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1762,7 +1761,7 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Squid Web Proxy 3.1.3:";; + short | recursive ) echo "Configuration of Squid Web Proxy 3.1.4:";; esac cat <<\_ACEOF @@ -1921,11 +1920,6 @@ build and install as part of the normal build process. For a list of available helpers see the helpers/external_acl directory. - --disable-mempools Disable memPools. Note that this option now simply - sets the default behaviour. Specific classes can - override this at runtime, and only lib/MemPool.c - needs to be altered to change the squid-wide default - for all classes. --enable-win32-service Compile Squid as a WIN32 Service. Works only on MS-Windows platforms (NT and up). --disable-unlinkd Do not use unlinkd @@ -2094,7 +2088,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Squid Web Proxy configure 3.1.3 +Squid Web Proxy configure 3.1.4 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -2108,7 +2102,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Squid Web Proxy $as_me 3.1.3, which was +It was created by Squid Web Proxy $as_me 3.1.4, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -2826,7 +2820,7 @@ # Define the identity of the package. PACKAGE='squid' - VERSION='3.1.3' + VERSION='3.1.4' cat >>confdefs.h <<_ACEOF @@ -5693,7 +5687,7 @@ ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5696 "configure"' > conftest.$ac_ext + echo '#line 5690 "configure"' > conftest.$ac_ext if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -8315,11 +8309,11 @@ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:8318: $lt_compile\"" >&5) + (eval echo "\"\$as_me:8312: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:8322: \$? = $ac_status" >&5 + echo "$as_me:8316: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -8605,11 +8599,11 @@ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:8608: $lt_compile\"" >&5) + (eval echo "\"\$as_me:8602: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:8612: \$? = $ac_status" >&5 + echo "$as_me:8606: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -8709,11 +8703,11 @@ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:8712: $lt_compile\"" >&5) + (eval echo "\"\$as_me:8706: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:8716: \$? = $ac_status" >&5 + echo "$as_me:8710: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -11109,7 +11103,7 @@ lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext < conftest.$ac_ext <&5) + (eval echo "\"\$as_me:13615: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:13625: \$? = $ac_status" >&5 + echo "$as_me:13619: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -13722,11 +13716,11 @@ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:13725: $lt_compile\"" >&5) + (eval echo "\"\$as_me:13719: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:13729: \$? = $ac_status" >&5 + echo "$as_me:13723: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -15305,11 +15299,11 @@ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:15308: $lt_compile\"" >&5) + (eval echo "\"\$as_me:15302: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:15312: \$? = $ac_status" >&5 + echo "$as_me:15306: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -15409,11 +15403,11 @@ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:15412: $lt_compile\"" >&5) + (eval echo "\"\$as_me:15406: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:15416: \$? = $ac_status" >&5 + echo "$as_me:15410: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -17624,11 +17618,11 @@ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:17627: $lt_compile\"" >&5) + (eval echo "\"\$as_me:17621: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:17631: \$? = $ac_status" >&5 + echo "$as_me:17625: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -17914,11 +17908,11 @@ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:17917: $lt_compile\"" >&5) + (eval echo "\"\$as_me:17911: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:17921: \$? = $ac_status" >&5 + echo "$as_me:17915: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -18018,11 +18012,11 @@ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:18021: $lt_compile\"" >&5) + (eval echo "\"\$as_me:18015: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:18025: \$? = $ac_status" >&5 + echo "$as_me:18019: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -26138,33 +26132,26 @@ fi -# Check whether --enable-mempools was given. -if test "${enable_mempools+set}" = set; then - enableval=$enable_mempools; if test "$enableval" = "no" ; then - { $as_echo "$as_me:$LINENO: memPools disabled" >&5 -$as_echo "$as_me: memPools disabled" >&6;} - -cat >>confdefs.h <<\_ACEOF -#define DISABLE_POOLS 1 -_ACEOF - - else - -cat >>confdefs.h <<\_ACEOF -#define DISABLE_POOLS 0 -_ACEOF - - fi - -else +#AC_ARG_ENABLE(chunkedmempools, +# AS_HELP_STRING([--enable-chunkedmempools], +# [Enable experimental chunked memPools. Note that this option +# simply sets the default behaviour. Specific classes can override this +# at runtime, and only lib/MemPool.c needs to be altered +# to change the squid-wide default for all classes.]), [ +# if test "x$enableval" = "xyes" ; then +# AC_DEFINE(USE_CHUNKEDMEMPOOLS, 1, [Define to Enable chunked Memory Pools support (experimental)]) +# else +# AC_MSG_NOTICE([chunked memPools disabled]) +# AC_DEFINE(USE_CHUNKEDMEMPOOLS, 0, [Define to Enable chunked Memory Pools support (experimental)]) +# fi +#], +#[ AC_MSG_NOTICE([chunked memPools disabled]) cat >>confdefs.h <<\_ACEOF -#define DISABLE_POOLS 0 +#define USE_CHUNKEDMEMPOOLS 0 _ACEOF - -fi - +#]) # Check whether --enable-win32-service was given. if test "${enable_win32_service+set}" = set; then @@ -47749,6 +47736,74 @@ DBLIB= + +{ $as_echo "$as_me:$LINENO: checking whether dbopen is declared" >&5 +$as_echo_n "checking whether dbopen is declared... " >&6; } +if test "${ac_cv_have_decl_dbopen+set}" = set; then + $as_echo_n "(cached) " >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +#if HAVE_SYS_TYPES_H +#include +#endif +#if HAVE_LIMITS_H +#include +#endif +#if HAVE_DB_185_H +#include +#elif HAVE_DB_H +#include +#endif + +int +main () +{ +#ifndef dbopen + (void) dbopen; +#endif + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_cv_have_decl_dbopen=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_cv_have_decl_dbopen=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_have_decl_dbopen" >&5 +$as_echo "$ac_cv_have_decl_dbopen" >&6; } + + { $as_echo "$as_me:$LINENO: checking if dbopen needs -ldb" >&5 $as_echo_n "checking if dbopen needs -ldb... " >&6; } if test "${ac_cv_dbopen_libdb+set}" = set; then @@ -51146,7 +51201,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Squid Web Proxy $as_me 3.1.3, which was +This file was extended by Squid Web Proxy $as_me 3.1.4, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -51199,7 +51254,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -Squid Web Proxy config.status 3.1.3 +Squid Web Proxy config.status 3.1.4 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff -u -r -N squid-3.1.3/configure.in squid-3.1.4/configure.in --- squid-3.1.3/configure.in 2010-05-02 22:48:00.000000000 +1200 +++ squid-3.1.4/configure.in 2010-05-31 01:21:49.000000000 +1200 @@ -2,7 +2,7 @@ dnl dnl $Id$ dnl -AC_INIT([Squid Web Proxy],[3.1.3],[http://www.squid-cache.org/bugs/],[squid]) +AC_INIT([Squid Web Proxy],[3.1.4],[http://www.squid-cache.org/bugs/],[squid]) AC_PREREQ(2.61) AC_CONFIG_HEADERS([include/autoconf.h]) AC_CONFIG_AUX_DIR(cfgaux) @@ -1921,20 +1921,22 @@ ]) dnl Disable "memPools" code -AC_ARG_ENABLE(mempools, - AS_HELP_STRING([--disable-mempools],[Disable memPools. Note that this option now simply sets the - default behaviour. Specific classes can override this at runtime, and - only lib/MemPool.c needs to be altered to change the squid-wide - default for all classes.]), -[ if test "$enableval" = "no" ; then - AC_MSG_NOTICE([memPools disabled]) - AC_DEFINE(DISABLE_POOLS, 1, [Define if you have problems with memPools and want to disable Pools]) - else - AC_DEFINE(DISABLE_POOLS, 0, [Define if you have problems with memPools and want to disable Pools.]) - fi -], -[ AC_DEFINE(DISABLE_POOLS, 0, [Define if you have problems with memPools and want to disable Pools.]) -]) +#AC_ARG_ENABLE(chunkedmempools, +# AS_HELP_STRING([--enable-chunkedmempools], +# [Enable experimental chunked memPools. Note that this option +# simply sets the default behaviour. Specific classes can override this +# at runtime, and only lib/MemPool.c needs to be altered +# to change the squid-wide default for all classes.]), [ +# if test "x$enableval" = "xyes" ; then +# AC_DEFINE(USE_CHUNKEDMEMPOOLS, 1, [Define to Enable chunked Memory Pools support (experimental)]) +# else +# AC_MSG_NOTICE([chunked memPools disabled]) +# AC_DEFINE(USE_CHUNKEDMEMPOOLS, 0, [Define to Enable chunked Memory Pools support (experimental)]) +# fi +#], +#[ AC_MSG_NOTICE([chunked memPools disabled]) + AC_DEFINE(USE_CHUNKEDMEMPOOLS, 0, [Define to Enable chunked Memory Pools support (experimental)]) +#]) dnl Enable WIN32 Service compile mode AC_ARG_ENABLE(win32-service, @@ -2846,7 +2848,24 @@ AC_SUBST(LIB_LBER) dnl Check for libdb +dnl this is not fully functional if db.h is for a differend db version DBLIB= + +dnl check that dbopen is actually defined in the header +dnl FIXME: in case of failure undef db-related includes etc. +AC_CHECK_DECL(dbopen,,,[ +#if HAVE_SYS_TYPES_H +#include +#endif +#if HAVE_LIMITS_H +#include +#endif +#if HAVE_DB_185_H +#include +#elif HAVE_DB_H +#include +#endif]) + dnl 1.85 AC_CACHE_CHECK(if dbopen needs -ldb,ac_cv_dbopen_libdb, [ SAVED_LIBS="$LIBS"; LIBS="$LIBS -ldb" diff -u -r -N squid-3.1.3/doc/debug-sections.txt squid-3.1.4/doc/debug-sections.txt --- squid-3.1.3/doc/debug-sections.txt 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/doc/debug-sections.txt 2010-05-31 01:21:11.000000000 +1200 @@ -1,37 +1,33 @@ - * DEBUG: -none -none Common Removal Policy -none ETag parsing support -none FDE -none Generate squid.conf.default and cf_parser.h -none Linked list functions (deprecated) -none LRU Removal Policy -none Memory Allocation -none Refcount allocator -section 0 Announcement Server -section 0 Client Database -section 0 Debug Routines -section 0 DNS Resolver -section 0 Hash Tables -section 0 UFS Store Dump +section -- CGI Cache Manager section -- External DISKD process implementation. -section 1 Main Loop -section 1 Startup and Main Loop -section 2 Unlink Daemon -section 3 Configuration File Parsing -section 4 Error Generation -section 5 Comms -section 5 Socket Functions -section 6 Disk I/O Routines -section 7 Multicast -section 8 Swap File Bitmap -section 9 File Transfer Protocol (FTP) +section -- Refcount allocator +section -- Unlink Daemon +section -- WWW Client +section 00 Announcement Server +section 00 Client Database +section 00 DNS Resolver Daemon +section 00 Debug Routines +section 00 Hash Tables +section 00 UFS Store Dump Tool +section 01 Main Loop +section 01 Startup and Main Loop +section 02 Unlink Daemon +section 03 Configuration File Parsing +section 03 Configuration Settings +section 04 Error Generation +section 05 Comm +section 05 Socket Functions +section 06 Disk I/O Routines +section 07 Multicast +section 08 Swap File Bitmap +section 09 File Transfer Protocol (FTP) section 10 Gopher section 11 Hypertext Transfer Protocol (HTTP) -section 12 Internet Cache Protocol +section 12 Internet Cache Protocol (ICP) section 13 High Level Memory Pool Management section 14 IP Cache +section 14 IP Storage and Handling section 15 Neighbor Routines section 16 Cache Manager Objects section 17 Request Forwarding @@ -53,13 +49,14 @@ section 22 Refresh Calculation section 23 URL Parsing section 23 URL Scheme parsing -section 25 MIME Parsing +section 25 MIME Parsing and Internal Icons +section 25 MiME Header Parsing section 26 Secure Sockets Layer Proxy section 27 Cache Announcer section 28 Access Control section 29 Authenticator -section 29 Negotiate Authenticator section 29 NTLM Authenticator +section 29 Negotiate Authenticator section 30 Ident (RFC 931) section 31 Hypertext Caching Protocol section 32 Asynchronous Disk I/O @@ -69,6 +66,8 @@ section 37 ICMP Routines section 38 Network Measurement Database section 39 Cache Array Routing Protocol +section 39 Peer source hash based selection +section 39 Peer user hash based selection section 40 Referer Logging section 40 User-Agent Logging section 41 Event Processing @@ -113,6 +112,7 @@ section 75 WHOIS protocol section 76 Internal Squid Object handling section 77 Delay Pools +section 78 DNS lookups section 78 DNS lookups; interacts with lib/rfc1035.c section 79 Disk IO Routines section 79 Squid-side DISKD I/O functions. @@ -120,18 +120,21 @@ section 79 Storage Manager COSS Interface section 79 Storage Manager UFS Interface section 80 WCCP Support -section 81 aio_xxx() POSIX emulation on Windows section 81 CPU Profiling Routines section 81 Store HEAP Removal Policies +section 81 aio_xxx() POSIX emulation on Windows section 82 External ACL section 83 SSL accelerator support section 84 Helper process maintenance section 85 Client-side Request Routines +section 86 ESI Expressions section 86 ESI processing section 87 Client-side Stream routines. section 88 Client-side Reply Routines -section 89 NAT / IP Interception +section 89 NAT / IP Interception section 90 HTTP Cache Control Header section 90 Storage Manager Client-Side Interface section 92 Storage File System +section 93 Adaptation section 93 ICAP (RFC 3507) Client +section 93 eCAP Interface diff -u -r -N squid-3.1.3/helpers/basic_auth/DB/Makefile.am squid-3.1.4/helpers/basic_auth/DB/Makefile.am --- squid-3.1.3/helpers/basic_auth/DB/Makefile.am 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/helpers/basic_auth/DB/Makefile.am 2010-05-31 01:21:11.000000000 +1200 @@ -15,7 +15,7 @@ config.test squid_db_auth.8: squid_db_auth - pod2man squid_db_auth squid_db_auth.8 + pod2man -s 8 squid_db_auth squid_db_auth.8 squid_db_auth: squid_db_auth.in $(do_subst) < $(srcdir)/squid_db_auth.in >squid_db_auth diff -u -r -N squid-3.1.3/helpers/basic_auth/DB/Makefile.in squid-3.1.4/helpers/basic_auth/DB/Makefile.in --- squid-3.1.3/helpers/basic_auth/DB/Makefile.in 2010-05-02 22:47:27.000000000 +1200 +++ squid-3.1.4/helpers/basic_auth/DB/Makefile.in 2010-05-31 01:21:26.000000000 +1200 @@ -485,7 +485,7 @@ squid_db_auth.8: squid_db_auth - pod2man squid_db_auth squid_db_auth.8 + pod2man -s 8 squid_db_auth squid_db_auth.8 squid_db_auth: squid_db_auth.in $(do_subst) < $(srcdir)/squid_db_auth.in >squid_db_auth diff -u -r -N squid-3.1.3/helpers/basic_auth/DB/squid_db_auth.in squid-3.1.4/helpers/basic_auth/DB/squid_db_auth.in --- squid-3.1.3/helpers/basic_auth/DB/squid_db_auth.in 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/helpers/basic_auth/DB/squid_db_auth.in 2010-05-31 01:21:12.000000000 +1200 @@ -3,13 +3,14 @@ use DBI; use Getopt::Long; use Pod::Usage; +use Digest::MD5 qw(md5 md5_hex md5_base64); $|=1; =pod =head1 NAME -db_auth.pl - Database auth helper for Squid +squid_db_auth - Database auth helper for Squid =cut @@ -22,12 +23,15 @@ my $db_cond = "enabled = 1"; my $plaintext = 0; my $persist = 0; +my $isjoomla = 0; +my $debug = 0; +my $hashsalt = undef; =pod =head1 SYNOPSIS -db_auth.pl [options] +squid_db_auth [options] =head1 DESCRIPTOIN @@ -62,15 +66,25 @@ =item B<--cond> Condition, defaults to enabled=1. Specify 1 or "" for no condition +If you use --joomla flag, this condition will be changed to block=0 =item B<--plaintext> Database contains plain-text passwords +=item B<--salt> + +Selects the correct salt to evaluate passwords + =item B<--persist> Keep a persistent database connection open between queries. +=item B<--joomla> + +Tells helper that user database is Joomla DB. So their unusual salt +hashing is understood. + =back =cut @@ -85,9 +99,13 @@ 'cond=s' => \$db_cond, 'plaintext' => \$plaintext, 'persist' => \$persist, + 'joomla' => \$isjoomla, + 'debug' => \$debug, + 'salt=s' => \$hashsalt, ); my ($_dbh, $_sth); +$db_cond = "block = 0" if $isjoomla; sub close_db() { @@ -105,7 +123,9 @@ warn ("Could not connect to $dsn\n"); return undef; } - $_sth = $_dbh->prepare("SELECT $db_passwdcol FROM $db_table WHERE $db_usercol = ?" . ($db_cond ne "" ? " AND $db_cond" : "")) || die; + my $sql_query; + $sql_query = "SELECT $db_passwdcol FROM $db_table WHERE $db_usercol = ?" . ($db_cond ne "" ? " AND $db_cond" : ""); + $_sth = $_dbh->prepare($sql_query) || die; return $_sth; } @@ -113,9 +133,17 @@ { my ($password, $key) = @_; - return 1 if crypt($password, $key) eq $key; - - return 1 if $plaintext && $password eq $key; + if ($isjoomla){ + my $salt; + my $key2; + ($key2,$salt) = split (/$salt/, $key); + return 1 if md5_hex($password.$salt).':'.$salt eq $key; + } + else{ + return 1 if defined $hashsalt && crypt($password, $hashsalt) eq $key; + return 1 if crypt($password, $key) eq $key; + return 1 if $plaintext && $password eq $key; + } return 0; } @@ -155,6 +183,7 @@ =head1 COPYRIGHT Copyright (C) 2007 Henrik Nordstrom +Copyright (C) 2010 Luis Daniel Lucio Quiroz (Joomla support) This program is free software. You may redistribute copies of it under the terms of the GNU General Public License version 2, or (at youropinion) any later version. diff -u -r -N squid-3.1.3/helpers/basic_auth/LDAP/squid_ldap_auth.8 squid-3.1.4/helpers/basic_auth/LDAP/squid_ldap_auth.8 --- squid-3.1.3/helpers/basic_auth/LDAP/squid_ldap_auth.8 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/helpers/basic_auth/LDAP/squid_ldap_auth.8 2010-05-31 01:21:12.000000000 +1200 @@ -146,7 +146,7 @@ . .TP .BI -v " 2|3" -LDAP protocol version. Defaults to 2 if not specified. +LDAP protocol version. Defaults to 3 if not specified. . .TP .BI -Z diff -u -r -N squid-3.1.3/helpers/basic_auth/LDAP/squid_ldap_auth.c squid-3.1.4/helpers/basic_auth/LDAP/squid_ldap_auth.c --- squid-3.1.3/helpers/basic_auth/LDAP/squid_ldap_auth.c 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/helpers/basic_auth/LDAP/squid_ldap_auth.c 2010-05-31 01:21:12.000000000 +1200 @@ -286,7 +286,7 @@ #ifdef LDAP_VERSION3 if (version == -1) { - version = LDAP_VERSION2; + version = LDAP_VERSION3; } if (ldap_set_option(ld, LDAP_OPT_PROTOCOL_VERSION, &version) != LDAP_SUCCESS) { fprintf(stderr, "Could not set LDAP_OPT_PROTOCOL_VERSION %d\n", diff -u -r -N squid-3.1.3/helpers/basic_auth/PAM/pam_auth.c squid-3.1.4/helpers/basic_auth/PAM/pam_auth.c --- squid-3.1.3/helpers/basic_auth/PAM/pam_auth.c 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/helpers/basic_auth/PAM/pam_auth.c 2010-05-31 01:21:11.000000000 +1200 @@ -100,7 +100,11 @@ * expects a single converstation message of type PAM_PROMPT_ECHO_OFF. */ static int +#if _SQUID_SOLARIS_ +password_conversation(int num_msg, struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) +#else password_conversation(int num_msg, const struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) +#endif { if (num_msg != 1 || msg[0]->msg_style != PAM_PROMPT_ECHO_OFF) { fprintf(stderr, "ERROR: Unexpected PAM converstaion '%d/%s'\n", msg[0]->msg_style, msg[0]->msg); diff -u -r -N squid-3.1.3/helpers/basic_auth/squid_radius_auth/radius-util.c squid-3.1.4/helpers/basic_auth/squid_radius_auth/radius-util.c --- squid-3.1.3/helpers/basic_auth/squid_radius_auth/radius-util.c 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/helpers/basic_auth/squid_radius_auth/radius-util.c 2010-05-31 01:21:11.000000000 +1200 @@ -90,7 +90,7 @@ if (*addr == '.') { dot_count++; digit_count = 0; - } else if (!isdigit(*addr)) { + } else if (!isdigit((int)*addr)) { dot_count = 5; } else { digit_count++; @@ -126,7 +126,7 @@ count = 0; *ptr = '\0'; while (*ip_str != '.' && *ip_str != '\0' && count < 4) { - if (!isdigit(*ip_str)) { + if (!isdigit((int)*ip_str)) { return((u_int32_t)0); } *ptr++ = *ip_str++; diff -u -r -N squid-3.1.3/helpers/basic_auth/squid_radius_auth/squid_rad_auth.c squid-3.1.4/helpers/basic_auth/squid_radius_auth/squid_rad_auth.c --- squid-3.1.3/helpers/basic_auth/squid_radius_auth/squid_rad_auth.c 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/helpers/basic_auth/squid_radius_auth/squid_rad_auth.c 2010-05-31 01:21:11.000000000 +1200 @@ -572,7 +572,7 @@ /* Parse out the username and password */ ptr = authstring; - while (isspace(*ptr)) + while (isspace((int)*ptr)) ptr++; if ((end = strchr(ptr, ' ')) == NULL) { printf("ERR\n"); /* No password */ @@ -581,7 +581,7 @@ *end = '\0'; urldecode(username, ptr, MAXPWNAM); ptr = end + 1; - while (isspace(*ptr)) + while (isspace((int)*ptr)) ptr++; urldecode(passwd, ptr, MAXPASS); diff -u -r -N squid-3.1.3/helpers/digest_auth/eDirectory/ldap_backend.c squid-3.1.4/helpers/digest_auth/eDirectory/ldap_backend.c --- squid-3.1.3/helpers/digest_auth/eDirectory/ldap_backend.c 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/helpers/digest_auth/eDirectory/ldap_backend.c 2010-05-31 01:21:11.000000000 +1200 @@ -61,7 +61,7 @@ static int searchscope = LDAP_SCOPE_SUBTREE; static int persistent = 0; static int noreferrals = 0; -static int debug = 0; +static int show_debug_messages = 0; static int port = LDAP_PORT; static int strip_nt_domain = 0; static int edir_universal_passwd = 0; @@ -209,7 +209,7 @@ snprintf(filter, sizeof(filter), usersearchfilter, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login); retrysrch: - if (debug) + if (show_debug_messages) fprintf(stderr, "user filter '%s', searchbase '%s'\n", filter, searchbase); rc = ldap_search_s(ld, searchbase, searchscope, filter, NULL, 0, &res); @@ -245,14 +245,14 @@ sprintf(searchbase, "%s=%s, %s", userdnattr, login, userbasedn); retrydnattr: - if (debug) + if (show_debug_messages) fprintf(stderr, "searchbase '%s'\n", searchbase); rc = ldap_search_s(ld, searchbase, searchscope, NULL, NULL, 0, &res); } if (rc == LDAP_SUCCESS) { entry = ldap_first_entry(ld, res); if (entry) { - if (debug) + if (show_debug_messages) printf("ldap dn: %s\n", ldap_get_dn(ld, entry)); if (edir_universal_passwd) { @@ -264,11 +264,11 @@ /* actually talk to NMAS to get a password */ nmas_res = nds_get_password(ld, ldap_get_dn(ld, entry), &universal_password_len, universal_password); if (nmas_res == LDAP_SUCCESS && universal_password) { - if (debug) + if (show_debug_messages) printf("NMAS returned value %s\n", universal_password); values[0] = universal_password; } else { - if (debug) + if (show_debug_messages) printf("Error reading Universal Password: %d = %s\n", nmas_res, ldap_err2string(nmas_res)); } } else { @@ -279,7 +279,7 @@ return NULL; } if (!values) { - if (debug) + if (show_debug_messages) printf("No attribute value found\n"); if (edir_universal_passwd) free(universal_password); @@ -299,7 +299,7 @@ } value++; } - if (debug) + if (show_debug_messages) printf("password: %s\n", password); if (password) password = strdup(password); @@ -417,7 +417,7 @@ ld = NULL; } } - if (debug) + if (show_debug_messages) fprintf(stderr, "Connected OK\n"); } } @@ -574,7 +574,7 @@ break; #endif case 'd': - debug = 1; + show_debug_messages = 1; break; case 'E': strip_nt_domain = 1; diff -u -r -N squid-3.1.3/helpers/digest_auth/ldap/ldap_backend.c squid-3.1.4/helpers/digest_auth/ldap/ldap_backend.c --- squid-3.1.3/helpers/digest_auth/ldap/ldap_backend.c 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/helpers/digest_auth/ldap/ldap_backend.c 2010-05-31 01:21:11.000000000 +1200 @@ -60,7 +60,7 @@ static int searchscope = LDAP_SCOPE_SUBTREE; static int persistent = 0; static int noreferrals = 0; -static int debug = 0; +static int show_debug_messages = 0; static int port = LDAP_PORT; static int strip_nt_domain = 0; static int aliasderef = LDAP_DEREF_NEVER; @@ -204,7 +204,7 @@ snprintf(filter, sizeof(filter), usersearchfilter, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login); retrysrch: - if (debug) + if (show_debug_messages) fprintf(stderr, "user filter '%s', searchbase '%s'\n", filter, searchbase); rc = ldap_search_s(ld, searchbase, searchscope, filter, NULL, 0, &res); @@ -240,7 +240,7 @@ sprintf(searchbase, "%s=%s, %s", userdnattr, login, userbasedn); retrydnattr: - if (debug) + if (show_debug_messages) fprintf(stderr, "searchbase '%s'\n", searchbase); rc = ldap_search_s(ld, searchbase, searchscope, NULL, NULL, 0, &res); } @@ -253,7 +253,7 @@ return NULL; } if (!values) { - if (debug) + if (show_debug_messages) printf("No attribute value found\n"); ldap_msgfree(res); return NULL; @@ -271,7 +271,7 @@ } value++; } - if (debug) + if (show_debug_messages) printf("password: %s\n", password); if (password) password = strdup(password); @@ -386,7 +386,7 @@ ld = NULL; } } - if (debug) + if (show_debug_messages) fprintf(stderr, "Connected OK\n"); } } @@ -541,7 +541,7 @@ break; #endif case 'd': - debug = 1; + show_debug_messages = 1; break; case 'E': strip_nt_domain = 1; diff -u -r -N squid-3.1.3/helpers/external_acl/ldap_group/squid_ldap_group.8 squid-3.1.4/helpers/external_acl/ldap_group/squid_ldap_group.8 --- squid-3.1.3/helpers/external_acl/ldap_group/squid_ldap_group.8 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/helpers/external_acl/ldap_group/squid_ldap_group.8 2010-05-31 01:21:11.000000000 +1200 @@ -129,7 +129,7 @@ . .TP .BI -v " 2|3" -LDAP protocol version. Defaults to 2 if not specified. +LDAP protocol version. Defaults to 3 if not specified. . .TP .BI -Z diff -u -r -N squid-3.1.3/helpers/external_acl/ldap_group/squid_ldap_group.c squid-3.1.4/helpers/external_acl/ldap_group/squid_ldap_group.c --- squid-3.1.3/helpers/external_acl/ldap_group/squid_ldap_group.c 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/helpers/external_acl/ldap_group/squid_ldap_group.c 2010-05-31 01:21:11.000000000 +1200 @@ -96,7 +96,7 @@ static int searchscope = LDAP_SCOPE_SUBTREE; static int persistent = 0; static int noreferrals = 0; -static int debug = 0; +static int show_debug_messages = 0; static int aliasderef = LDAP_DEREF_NEVER; #if defined(NETSCAPE_SSL) static char *sslpath = NULL; @@ -367,7 +367,7 @@ break; #endif case 'd': - debug = 1; + show_debug_messages = 1; break; case 'g': use_extension_dn = 1; @@ -534,7 +534,7 @@ #ifdef LDAP_VERSION3 if (version == -1) { - version = LDAP_VERSION2; + version = LDAP_VERSION3; } if (ldap_set_option(ld, LDAP_OPT_PROTOCOL_VERSION, &version) != LDAP_SUCCESS) { fprintf(stderr, "Could not set LDAP_OPT_PROTOCOL_VERSION %d\n", @@ -572,7 +572,7 @@ break; } } - if (debug) + if (show_debug_messages) fprintf(stderr, "Connected OK\n"); } if (searchLDAP(ld, group, user, extension_dn) == 0) { @@ -706,7 +706,7 @@ fprintf(stderr, PROGRAM_NAME " ERROR, Failed to construct LDAP search filter. filter=\"%s\", user=\"%s\", group=\"%s\"\n", filter, member, group); return 1; } - if (debug) + if (show_debug_messages) fprintf(stderr, "group filter '%s', searchbase '%s'\n", filter, searchbase); rc = ldap_search_s(ld, searchbase, searchscope, filter, searchattr, 1, &res); @@ -755,7 +755,7 @@ snprintf(searchbase, sizeof(searchbase), "%s", userbasedn ? userbasedn : basedn); ldap_escape_value(escaped_login, sizeof(escaped_login), login); snprintf(filter, sizeof(filter), usersearchfilter, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login, escaped_login); - if (debug) + if (show_debug_messages) fprintf(stderr, "user filter '%s', searchbase '%s'\n", filter, searchbase); rc = ldap_search_s(ld, searchbase, searchscope, filter, searchattr, 1, &res); if (rc != LDAP_SUCCESS) { diff -u -r -N squid-3.1.3/helpers/external_acl/session/config.test squid-3.1.4/helpers/external_acl/session/config.test --- squid-3.1.3/helpers/external_acl/session/config.test 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/helpers/external_acl/session/config.test 2010-05-31 01:21:11.000000000 +1200 @@ -4,7 +4,7 @@ if [ -f /usr/include/db_185.h ]; then exit 0 fi -if [ -f /usr/include/db.h ]; then +if [ -f /usr/include/db.h ] && grep dbopen /usr/include/db.h; then exit 0 fi exit 1 diff -u -r -N squid-3.1.3/helpers/external_acl/session/squid_session.c squid-3.1.4/helpers/external_acl/session/squid_session.c --- squid-3.1.3/helpers/external_acl/session/squid_session.c 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/helpers/external_acl/session/squid_session.c 2010-05-31 01:21:12.000000000 +1200 @@ -43,12 +43,11 @@ #ifndef __BIT_TYPES_DEFINED__ #define __BIT_TYPES_DEFINED__ #endif -#if defined(HAVE_DB_185_H) + +#if HAVE_DB_185_H #include -#elif defined(HAVE_DB_H) +#elif HAVE_DB_H #include -#else -#include #endif static int session_ttl = 3600; diff -u -r -N squid-3.1.3/helpers/ntlm_auth/smb_lm/smbval/rfcnb-util.c squid-3.1.4/helpers/ntlm_auth/smb_lm/smbval/rfcnb-util.c --- squid-3.1.3/helpers/ntlm_auth/smb_lm/smbval/rfcnb-util.c 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/helpers/ntlm_auth/smb_lm/smbval/rfcnb-util.c 2010-05-31 01:21:11.000000000 +1200 @@ -23,12 +23,13 @@ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ -#include -#include +#include "config.h" #include "std-includes.h" #include "rfcnb-priv.h" #include "rfcnb-util.h" +#include +#include #include "rfcnb-io.h" #include diff -u -r -N squid-3.1.3/helpers/ntlm_auth/smb_lm/smbval/session.c squid-3.1.4/helpers/ntlm_auth/smb_lm/smbval/session.c --- squid-3.1.3/helpers/ntlm_auth/smb_lm/smbval/session.c 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/helpers/ntlm_auth/smb_lm/smbval/session.c 2010-05-31 01:21:11.000000000 +1200 @@ -23,8 +23,7 @@ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ -#include -#include +#include "config.h" int RFCNB_errno = 0; int RFCNB_saved_errno = 0; @@ -34,6 +33,8 @@ #include #include "rfcnb-priv.h" #include "rfcnb-util.h" +#include +#include #include "rfcnb-io.h" #include "rfcnb.h" diff -u -r -N squid-3.1.3/include/autoconf.h.in squid-3.1.4/include/autoconf.h.in --- squid-3.1.3/include/autoconf.h.in 2010-05-02 22:47:25.000000000 +1200 +++ squid-3.1.4/include/autoconf.h.in 2010-05-31 01:21:24.000000000 +1200 @@ -36,9 +36,6 @@ /* Traffic management via "delay pools". */ #undef DELAY_POOLS -/* Define if you have problems with memPools and want to disable Pools. */ -#undef DISABLE_POOLS - /* Enable following X-Forwarded-For headers */ #undef FOLLOW_X_FORWARDED_FOR @@ -1025,6 +1022,9 @@ still semi-experimental. */ #undef USE_CACHE_DIGESTS +/* Define to Enable chunked Memory Pools support (experimental) */ +#undef USE_CHUNKEDMEMPOOLS + /* DiskIO modules are expected to be available. */ #undef USE_DISKIO diff -u -r -N squid-3.1.3/include/MemPoolChunked.h squid-3.1.4/include/MemPoolChunked.h --- squid-3.1.3/include/MemPoolChunked.h 1970-01-01 12:00:00.000000000 +1200 +++ squid-3.1.4/include/MemPoolChunked.h 2010-05-31 01:21:11.000000000 +1200 @@ -0,0 +1,82 @@ +#ifndef _MEM_POOL_CHUNKED_H_ +#define _MEM_POOL_CHUNKED_H_ + +#include "MemPool.h" + +/// \ingroup MemPoolsAPI +#define MEM_PAGE_SIZE 4096 +/// \ingroup MemPoolsAPI +#define MEM_CHUNK_SIZE 4096 * 4 +/// \ingroup MemPoolsAPI +#define MEM_CHUNK_MAX_SIZE 256 * 1024 /* 2MB */ +/// \ingroup MemPoolsAPI +#define MEM_MIN_FREE 32 +/// \ingroup MemPoolsAPI +#define MEM_MAX_FREE 65535 /* ushort is max number of items per chunk */ + +class MemChunk; + +/// \ingroup MemPoolsAPI +class MemPoolChunked : public MemImplementingAllocator +{ +public: + friend class MemChunk; + MemPoolChunked(const char *label, size_t obj_size); + ~MemPoolChunked(); + void convertFreeCacheToChunkFreeCache(); + virtual void clean(time_t maxage); + + /** + \param stats Object to be filled with statistical data about pool. + \retval Number of objects in use, ie. allocated. + */ + virtual int getStats(MemPoolStats * stats, int accumulate); + + void createChunk(); + void *get(); + void push(void *obj); + virtual int getInUseCount(); +protected: + virtual void *allocate(); + virtual void deallocate(void *, bool aggressive); +public: + /** + * Allows you tune chunk size of pooling. Objects are allocated in chunks + * instead of individually. This conserves memory, reduces fragmentation. + * Because of that memory can be freed also only in chunks. Therefore + * there is tradeoff between memory conservation due to chunking and free + * memory fragmentation. + * + \note As a general guideline, increase chunk size only for pools that keep + * very many items for relatively long time. + */ + virtual void setChunkSize(size_t chunksize); + + virtual bool idleTrigger(int shift) const; + + size_t chunk_size; + int chunk_capacity; + int memPID; + int chunkCount; + void *freeCache; + MemChunk *nextFreeChunk; + MemChunk *Chunks; + Splay allChunks; +}; + +/// \ingroup MemPoolsAPI +class MemChunk +{ +public: + MemChunk(MemPoolChunked *pool); + ~MemChunk(); + void *freeList; + void *objCache; + int inuse_count; + MemChunk *nextFreeChunk; + MemChunk *next; + time_t lastref; + MemPoolChunked *pool; +}; + +#endif /* _MEM_POOL_CHUNKED_H_ */ diff -u -r -N squid-3.1.3/include/MemPool.h squid-3.1.4/include/MemPool.h --- squid-3.1.3/include/MemPool.h 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/include/MemPool.h 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ -#ifndef _MEM_POOLS_H_ -#define _MEM_POOLS_H_ +#ifndef _MEM_POOL_H_ +#define _MEM_POOL_H_ /** \defgroup MemPoolsAPI Memory Management (Memory Pool Allocator) @@ -45,8 +45,6 @@ /// \ingroup MemPoolsAPI #define MB ((size_t)1024*1024) /// \ingroup MemPoolsAPI -#define mem_unlimited_size 2 * 1024 * MB -/// \ingroup MemPoolsAPI #define toMB(size) ( ((double) size) / MB ) /// \ingroup MemPoolsAPI #define toKB(size) ( (size + 1024 - 1) / 1024 ) @@ -63,7 +61,6 @@ #define MEM_MAX_FREE 65535 /* ushort is max number of items per chunk */ class MemImplementingAllocator; -class MemChunk; class MemPoolStats; /// \ingroup MemPoolsAPI @@ -97,16 +94,19 @@ class MemPoolMeter { public: + MemPoolMeter(); void flush(); MemMeter alloc; MemMeter inuse; MemMeter idle; - /** account Allocations */ - mgb_t gb_saved; /** history Allocations */ - mgb_t gb_osaved; + mgb_t gb_allocated; + mgb_t gb_oallocated; + + /** account Saved Allocations */ + mgb_t gb_saved; /** account Free calls */ mgb_t gb_freed; @@ -130,21 +130,13 @@ MemImplementingAllocator * create(const char *label, size_t obj_size); /** - \param label Name for the pool. Displayed in stats. - \param obj_size Size of elements in MemPool. - \param chunked ?? - */ - MemImplementingAllocator * create(const char *label, size_t obj_size, bool const chunked); - - /** * Sets upper limit in bytes to amount of free ram kept in pools. This is * not strict upper limit, but a hint. When MemPools are over this limit, * totally free chunks are immediately considered for release. Otherwise * only chunks that have not been referenced for a long time are checked. */ - void setIdleLimit(size_t new_idle_limit); - - size_t idleLimit() const; + void setIdleLimit(ssize_t new_idle_limit); + ssize_t idleLimit() const; /** \par @@ -176,7 +168,7 @@ void setDefaultPoolChunking(bool const &); MemImplementingAllocator *pools; - int mem_idle_limit; + ssize_t mem_idle_limit; int poolCount; bool defaultIsChunked; private: @@ -197,7 +189,7 @@ \param stats Object to be filled with statistical data about pool. \retval Number of objects in use, ie. allocated. */ - virtual int getStats(MemPoolStats * stats) = 0; + virtual int getStats(MemPoolStats * stats, int accumulate = 0) = 0; virtual MemPoolMeter const &getMeter() const = 0; @@ -325,6 +317,7 @@ { public: MemImplementingAllocator(char const *aLabel, size_t aSize); + virtual ~MemImplementingAllocator(); virtual MemPoolMeter const &getMeter() const; virtual MemPoolMeter &getMeter(); virtual void flushMetersFull(); @@ -342,111 +335,23 @@ virtual bool idleTrigger(int shift) const = 0; virtual void clean(time_t maxage) = 0; - /** Hint to the allocator - may be ignored */ - virtual void setChunkSize(size_t chunksize) {} virtual size_t objectSize() const; virtual int getInUseCount() = 0; protected: virtual void *allocate() = 0; - virtual void deallocate(void *) = 0; -private: + virtual void deallocate(void *, bool aggressive) = 0; MemPoolMeter meter; + int memPID; public: MemImplementingAllocator *next; public: size_t alloc_calls; size_t free_calls; + size_t saved_calls; size_t obj_size; }; /// \ingroup MemPoolsAPI -class MemPool : public MemImplementingAllocator -{ -public: - friend class MemChunk; - MemPool(const char *label, size_t obj_size); - ~MemPool(); - void convertFreeCacheToChunkFreeCache(); - virtual void clean(time_t maxage); - - /** - \param stats Object to be filled with statistical data about pool. - \retval Number of objects in use, ie. allocated. - */ - virtual int getStats(MemPoolStats * stats); - - void createChunk(); - void *get(); - void push(void *obj); - virtual int getInUseCount(); -protected: - virtual void *allocate(); - virtual void deallocate(void *); -public: - /** - * Allows you tune chunk size of pooling. Objects are allocated in chunks - * instead of individually. This conserves memory, reduces fragmentation. - * Because of that memory can be freed also only in chunks. Therefore - * there is tradeoff between memory conservation due to chunking and free - * memory fragmentation. - * - \note As a general guideline, increase chunk size only for pools that keep - * very many items for relatively long time. - */ - virtual void setChunkSize(size_t chunksize); - - virtual bool idleTrigger(int shift) const; - - size_t chunk_size; - int chunk_capacity; - int memPID; - int chunkCount; - size_t inuse; - size_t idle; - void *freeCache; - MemChunk *nextFreeChunk; - MemChunk *Chunks; - Splay allChunks; -}; - -/// \ingroup MemPoolsAPI -class MemMalloc : public MemImplementingAllocator -{ -public: - MemMalloc(char const *label, size_t aSize); - virtual bool idleTrigger(int shift) const; - virtual void clean(time_t maxage); - - /** - \param stats Object to be filled with statistical data about pool. - \retval Number of objects in use, ie. allocated. - */ - virtual int getStats(MemPoolStats * stats); - - virtual int getInUseCount(); -protected: - virtual void *allocate(); - virtual void deallocate(void *); -private: - int inuse; -}; - -/// \ingroup MemPoolsAPI -class MemChunk -{ -public: - MemChunk(MemPool *pool); - ~MemChunk(); - void *freeList; - void *objCache; - int inuse_count; - MemChunk *nextFreeChunk; - MemChunk *next; - time_t lastref; - MemPool *pool; -}; - -/// \ingroup MemPoolsAPI class MemPoolStats { public: @@ -488,7 +393,7 @@ int tot_items_idle; int tot_overhead; - int mem_idle_limit; + ssize_t mem_idle_limit; }; /// \ingroup MemPoolsAPI @@ -536,4 +441,4 @@ } -#endif /* _MEM_POOLS_H_ */ +#endif /* _MEM_POOL_H_ */ diff -u -r -N squid-3.1.3/include/MemPoolMalloc.h squid-3.1.4/include/MemPoolMalloc.h --- squid-3.1.3/include/MemPoolMalloc.h 1970-01-01 12:00:00.000000000 +1200 +++ squid-3.1.4/include/MemPoolMalloc.h 2010-05-31 01:21:11.000000000 +1200 @@ -0,0 +1,49 @@ +#ifndef _MEM_POOL_MALLOC_H_ +#define _MEM_POOL_MALLOC_H_ + +/** + \defgroup MemPoolsAPI Memory Management (Memory Pool Allocator) + \ingroup Components + * + *\par + * MemPools are a pooled memory allocator running on top of malloc(). It's + * purpose is to reduce memory fragmentation and provide detailed statistics + * on memory consumption. + * + \par + * Preferably all memory allocations in Squid should be done using MemPools + * or one of the types built on top of it (i.e. cbdata). + * + \note Usually it is better to use cbdata types as these gives you additional + * safeguards in references and typechecking. However, for high usage pools where + * the cbdata functionality of cbdata is not required directly using a MemPool + * might be the way to go. + */ + +#include "MemPool.h" + +/// \ingroup MemPoolsAPI +class MemPoolMalloc : public MemImplementingAllocator +{ +public: + MemPoolMalloc(char const *label, size_t aSize); + ~MemPoolMalloc(); + virtual bool idleTrigger(int shift) const; + virtual void clean(time_t maxage); + + /** + \param stats Object to be filled with statistical data about pool. + \retval Number of objects in use, ie. allocated. + */ + virtual int getStats(MemPoolStats * stats, int accumulate); + + virtual int getInUseCount(); +protected: + virtual void *allocate(); + virtual void deallocate(void *, bool aggressive); +private: + Stack freelist; +}; + + +#endif /* _MEM_POOL_MALLOC_H_ */ diff -u -r -N squid-3.1.3/include/RefCount.h squid-3.1.4/include/RefCount.h --- squid-3.1.3/include/RefCount.h 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/include/RefCount.h 2010-05-31 01:21:11.000000000 +1200 @@ -1,7 +1,7 @@ /* * $Id$ * - * DEBUG: none Refcount allocator + * DEBUG: section -- Refcount allocator * AUTHOR: Robert Collins * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/include/version.h squid-3.1.4/include/version.h --- squid-3.1.3/include/version.h 2010-05-02 22:48:00.000000000 +1200 +++ squid-3.1.4/include/version.h 2010-05-31 01:21:49.000000000 +1200 @@ -9,7 +9,7 @@ */ #ifndef SQUID_RELEASE_TIME -#define SQUID_RELEASE_TIME 1272797215 +#define SQUID_RELEASE_TIME 1275225661 #endif #ifndef APP_SHORTNAME diff -u -r -N squid-3.1.3/lib/hash.c squid-3.1.4/lib/hash.c --- squid-3.1.3/lib/hash.c 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/lib/hash.c 2010-05-31 01:21:11.000000000 +1200 @@ -2,7 +2,7 @@ /* * $Id$ * - * DEBUG: section 0 Hash Tables + * DEBUG: section 00 Hash Tables * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/lib/Makefile.am squid-3.1.4/lib/Makefile.am --- squid-3.1.3/lib/Makefile.am 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/lib/Makefile.am 2010-05-31 01:21:11.000000000 +1200 @@ -49,6 +49,8 @@ win32lib.c libmiscutil_a_SOURCES = \ MemPool.cc \ + MemPoolChunked.cc \ + MemPoolMalloc.cc \ base64.c \ charset.c \ getfullhostname.c \ diff -u -r -N squid-3.1.3/lib/Makefile.in squid-3.1.4/lib/Makefile.in --- squid-3.1.3/lib/Makefile.in 2010-05-02 22:47:34.000000000 +1200 +++ squid-3.1.4/lib/Makefile.in 2010-05-31 01:21:31.000000000 +1200 @@ -56,21 +56,23 @@ ARFLAGS = cru libmiscutil_a_AR = $(AR) $(ARFLAGS) libmiscutil_a_DEPENDENCIES = $(LIBOBJS) -am__libmiscutil_a_SOURCES_DIST = MemPool.cc base64.c charset.c \ - getfullhostname.c hash.c heap.c html_quote.c iso3307.c md5.c \ - radix.c rfc1035.c rfc1123.c rfc1738.c rfc2617.c rfc3596.c \ - Splay.cc strnstr.cc stub_memaccount.c util.c uudecode.c \ - xusleep.c Profiler.c win32lib.c +am__libmiscutil_a_SOURCES_DIST = MemPool.cc MemPoolChunked.cc \ + MemPoolMalloc.cc base64.c charset.c getfullhostname.c hash.c \ + heap.c html_quote.c iso3307.c md5.c radix.c rfc1035.c \ + rfc1123.c rfc1738.c rfc2617.c rfc3596.c Splay.cc strnstr.cc \ + stub_memaccount.c util.c uudecode.c xusleep.c Profiler.c \ + win32lib.c @ENABLE_XPROF_STATS_TRUE@am__objects_1 = Profiler.$(OBJEXT) @ENABLE_WIN32SPECIFIC_TRUE@am__objects_2 = win32lib.$(OBJEXT) -am_libmiscutil_a_OBJECTS = MemPool.$(OBJEXT) base64.$(OBJEXT) \ - charset.$(OBJEXT) getfullhostname.$(OBJEXT) hash.$(OBJEXT) \ - heap.$(OBJEXT) html_quote.$(OBJEXT) iso3307.$(OBJEXT) \ - md5.$(OBJEXT) radix.$(OBJEXT) rfc1035.$(OBJEXT) \ - rfc1123.$(OBJEXT) rfc1738.$(OBJEXT) rfc2617.$(OBJEXT) \ - rfc3596.$(OBJEXT) Splay.$(OBJEXT) strnstr.$(OBJEXT) \ - stub_memaccount.$(OBJEXT) util.$(OBJEXT) uudecode.$(OBJEXT) \ - xusleep.$(OBJEXT) $(am__objects_1) $(am__objects_2) +am_libmiscutil_a_OBJECTS = MemPool.$(OBJEXT) MemPoolChunked.$(OBJEXT) \ + MemPoolMalloc.$(OBJEXT) base64.$(OBJEXT) charset.$(OBJEXT) \ + getfullhostname.$(OBJEXT) hash.$(OBJEXT) heap.$(OBJEXT) \ + html_quote.$(OBJEXT) iso3307.$(OBJEXT) md5.$(OBJEXT) \ + radix.$(OBJEXT) rfc1035.$(OBJEXT) rfc1123.$(OBJEXT) \ + rfc1738.$(OBJEXT) rfc2617.$(OBJEXT) rfc3596.$(OBJEXT) \ + Splay.$(OBJEXT) strnstr.$(OBJEXT) stub_memaccount.$(OBJEXT) \ + util.$(OBJEXT) uudecode.$(OBJEXT) xusleep.$(OBJEXT) \ + $(am__objects_1) $(am__objects_2) libmiscutil_a_OBJECTS = $(am_libmiscutil_a_OBJECTS) libntlmauth_a_AR = $(AR) $(ARFLAGS) libntlmauth_a_DEPENDENCIES = $(LIBOBJS) @@ -352,6 +354,8 @@ libmiscutil_a_SOURCES = \ MemPool.cc \ + MemPoolChunked.cc \ + MemPoolMalloc.cc \ base64.c \ charset.c \ getfullhostname.c \ @@ -485,6 +489,8 @@ @AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/strtoll.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/tempnam.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MemPool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MemPoolChunked.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MemPoolMalloc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Profiler.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Splay.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/base64.Po@am__quote@ diff -u -r -N squid-3.1.3/lib/MemPool.cc squid-3.1.4/lib/MemPool.cc --- squid-3.1.3/lib/MemPool.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/lib/MemPool.cc 2010-05-31 01:21:11.000000000 +1200 @@ -33,63 +33,16 @@ * */ -/* - * Old way: - * xmalloc each item separately, upon free stack into idle pool array. - * each item is individually malloc()ed from system, imposing libmalloc - * overhead, and additionally we add our overhead of pointer size per item - * as we keep a list of pointer to free items. - * - * Chunking: - * xmalloc Chunk that fits at least MEM_MIN_FREE (32) items in an array, but - * limit Chunk size to MEM_CHUNK_MAX_SIZE (256K). Chunk size is rounded up to - * MEM_PAGE_SIZE (4K), trying to have chunks in multiples of VM_PAGE size. - * Minimum Chunk size is MEM_CHUNK_SIZE (16K). - * A number of items fits into a single chunk, depending on item size. - * Maximum number of items per chunk is limited to MEM_MAX_FREE (65535). - * - * We populate Chunk with a linkedlist, each node at first word of item, - * and pointing at next free item. Chunk->FreeList is pointing at first - * free node. Thus we stuff free housekeeping into the Chunk itself, and - * omit pointer overhead per item. - * - * Chunks are created on demand, and new chunks are inserted into linklist - * of chunks so that Chunks with smaller pointer value are placed closer - * to the linklist head. Head is a hotspot, servicing most of requests, so - * slow sorting occurs and Chunks in highest memory tend to become idle - * and freeable. - * - * event is registered that runs every 15 secs and checks reference time - * of each idle chunk. If a chunk is not referenced for 15 secs, it is - * released. - * - * [If mem_idle_limit is exceeded with pools, every chunk that becomes - * idle is immediately considered for release, unless this is the only - * chunk with free items in it.] (not implemented) - * - * In cachemgr output, there are new columns for chunking. Special item, - * Frag, is shown to estimate approximately fragmentation of chunked - * pools. Fragmentation is calculated by taking amount of items in use, - * calculating needed amount of chunks to fit all, and then comparing to - * actual amount of chunks in use. Frag number, in percent, is showing - * how many percent of chunks are in use excessively. 100% meaning that - * twice the needed amount of chunks are in use. - * "part" item shows number of chunks partially filled. This shows how - * badly fragmentation is spread across all chunks. - * - * Andres Kroonmaa. - * Copyright (c) 2003, Robert Collins - */ - #include "config.h" #if HAVE_ASSERT_H #include #endif #include "MemPool.h" +#include "MemPoolChunked.h" +#include "MemPoolMalloc.h" #define FLUSH_LIMIT 1000 /* Flush memPool counters to memMeters after flush limit calls */ -#define MEM_MAX_MMAP_CHUNKS 2048 #if HAVE_STRING_H #include @@ -107,10 +60,6 @@ static int Pool_id_counter = 0; -/* local prototypes */ -static int memCompChunks(MemChunk * const &, MemChunk * const &); -static int memCompObjChunks(void * const &, MemChunk * const &); - MemPools & MemPools::GetInstance() { @@ -154,264 +103,37 @@ } void -MemPools::setIdleLimit(size_t new_idle_limit) +MemPools::setIdleLimit(ssize_t new_idle_limit) { mem_idle_limit = new_idle_limit; } -size_t +ssize_t MemPools::idleLimit() const { return mem_idle_limit; } -/* Compare chunks */ -static int -memCompChunks(MemChunk * const &chunkA, MemChunk * const &chunkB) -{ - if (chunkA->objCache > chunkB->objCache) - return 1; - else if (chunkA->objCache < chunkB->objCache) - return -1; - else - return 0; -} - -/* Compare object to chunk */ -static int -memCompObjChunks(void *const &obj, MemChunk * const &chunk) -{ - /* object is lower in memory than the chunks arena */ - if (obj < chunk->objCache) - return -1; - /* object is within the pool */ - if (obj < (void *) ((char *) chunk->objCache + chunk->pool->chunk_size)) - return 0; - /* object is above the pool */ - return 1; -} - -MemChunk::MemChunk(MemPool *aPool) -{ - /* should have a pool for this too - - * note that this requres: - * allocate one chunk for the pool of chunks's first chunk - * allocate a chunk from that pool - * move the contents of one chunk into the other - * free the first chunk. - */ - inuse_count = 0; - next = NULL; - pool = aPool; - - objCache = xcalloc(1, pool->chunk_size); - freeList = objCache; - void **Free = (void **)freeList; - - for (int i = 1; i < pool->chunk_capacity; i++) { - *Free = (void *) ((char *) Free + pool->obj_size); - void **nextFree = (void **)*Free; - (void) VALGRIND_MAKE_MEM_NOACCESS(Free, pool->obj_size); - Free = nextFree; - } - nextFreeChunk = pool->nextFreeChunk; - pool->nextFreeChunk = this; - - memMeterAdd(pool->getMeter().alloc, pool->chunk_capacity); - memMeterAdd(pool->getMeter().idle, pool->chunk_capacity); - pool->idle += pool->chunk_capacity; - pool->chunkCount++; - lastref = squid_curtime; - pool->allChunks.insert(this, memCompChunks); -} - -MemPool::MemPool(const char *aLabel, size_t aSize) : MemImplementingAllocator(aLabel, aSize) -{ - chunk_size = 0; - chunk_capacity = 0; - memPID = 0; - chunkCount = 0; - inuse = 0; - idle = 0; - freeCache = 0; - nextFreeChunk = 0; - Chunks = 0; - next = 0; - MemImplementingAllocator *last_pool; - - assert(aLabel != NULL && aSize); - - setChunkSize(MEM_CHUNK_SIZE); - - /* Append as Last */ - for (last_pool = MemPools::GetInstance().pools; last_pool && last_pool->next;) - last_pool = last_pool->next; - if (last_pool) - last_pool->next = this; - else - MemPools::GetInstance().pools = this; - - memPID = ++Pool_id_counter; -} - -MemChunk::~MemChunk() -{ - memMeterDel(pool->getMeter().alloc, pool->chunk_capacity); - memMeterDel(pool->getMeter().idle, pool->chunk_capacity); - pool->idle -= pool->chunk_capacity; - pool->chunkCount--; - pool->allChunks.remove(this, memCompChunks); - xfree(objCache); -} - -void -MemPool::push(void *obj) -{ - void **Free; - /* XXX We should figure out a sane way of avoiding having to clear - * all buffers. For example data buffers such as used by MemBuf do - * not really need to be cleared.. There was a condition based on - * the object size here, but such condition is not safe. - */ - if (doZeroOnPush) - memset(obj, 0, obj_size); - Free = (void **)obj; - *Free = freeCache; - freeCache = obj; - (void) VALGRIND_MAKE_MEM_NOACCESS(obj, obj_size); -} - -/* - * Find a chunk with a free item. - * Create new chunk on demand if no chunk with frees found. - * Insert new chunk in front of lowest ram chunk, making it preferred in future, - * and resulting slow compaction towards lowest ram area. - */ -void * -MemPool::get() -{ - void **Free; - - /* first, try cache */ - if (freeCache) { - Free = (void **)freeCache; - (void) VALGRIND_MAKE_MEM_DEFINED(Free, obj_size); - freeCache = *Free; - *Free = NULL; - return Free; - } - /* then try perchunk freelist chain */ - if (nextFreeChunk == NULL) { - /* no chunk with frees, so create new one */ - createChunk(); - } - /* now we have some in perchunk freelist chain */ - MemChunk *chunk = nextFreeChunk; - - Free = (void **)chunk->freeList; - chunk->freeList = *Free; - *Free = NULL; - chunk->inuse_count++; - chunk->lastref = squid_curtime; - - if (chunk->freeList == NULL) { - /* last free in this chunk, so remove us from perchunk freelist chain */ - nextFreeChunk = chunk->nextFreeChunk; - } - (void) VALGRIND_MAKE_MEM_DEFINED(Free, obj_size); - return Free; -} - -/* just create a new chunk and place it into a good spot in the chunk chain */ -void -MemPool::createChunk() -{ - MemChunk *chunk, *newChunk; - - newChunk = new MemChunk(this); - - chunk = Chunks; - if (chunk == NULL) { /* first chunk in pool */ - Chunks = newChunk; - return; - } - if (newChunk->objCache < chunk->objCache) { - /* we are lowest ram chunk, insert as first chunk */ - newChunk->next = chunk; - Chunks = newChunk; - return; - } - while (chunk->next) { - if (newChunk->objCache < chunk->next->objCache) { - /* new chunk is in lower ram, insert here */ - newChunk->next = chunk->next; - chunk->next = newChunk; - return; - } - chunk = chunk->next; - } - /* we are the worst chunk in chain, add as last */ - chunk->next = newChunk; -} - /* Change the default calue of defaultIsChunked to override * all pools - including those used before main() starts where * MemPools::GetInstance().setDefaultPoolChunking() can be called. */ MemPools::MemPools() : pools(NULL), mem_idle_limit(2 * MB), - poolCount (0), defaultIsChunked (!DISABLE_POOLS && !RUNNING_ON_VALGRIND) + poolCount (0), defaultIsChunked (USE_CHUNKEDMEMPOOLS && !RUNNING_ON_VALGRIND) { char *cfg = getenv("MEMPOOLS"); if (cfg) defaultIsChunked = atoi(cfg); -#if HAVE_MALLOPT && M_MMAP_MAX - mallopt(M_MMAP_MAX, MEM_MAX_MMAP_CHUNKS); -#endif -} - -void -MemPool::setChunkSize(size_t chunksize) -{ - int cap; - size_t csize = chunksize; - - if (Chunks) /* unsafe to tamper */ - return; - - csize = ((csize + MEM_PAGE_SIZE - 1) / MEM_PAGE_SIZE) * MEM_PAGE_SIZE; /* round up to page size */ - cap = csize / obj_size; - - if (cap < MEM_MIN_FREE) - cap = MEM_MIN_FREE; - if (cap * obj_size > MEM_CHUNK_MAX_SIZE) - cap = MEM_CHUNK_MAX_SIZE / obj_size; - if (cap > MEM_MAX_FREE) - cap = MEM_MAX_FREE; - if (cap < 1) - cap = 1; - - csize = cap * obj_size; - csize = ((csize + MEM_PAGE_SIZE - 1) / MEM_PAGE_SIZE) * MEM_PAGE_SIZE; /* round up to page size */ - cap = csize / obj_size; - - chunk_capacity = cap; - chunk_size = csize; } MemImplementingAllocator * MemPools::create(const char *label, size_t obj_size) { - return create (label, obj_size, defaultIsChunked); -} - -MemImplementingAllocator * -MemPools::create(const char *label, size_t obj_size, bool const chunked) -{ ++poolCount; - if (chunked) - return new MemPool (label, obj_size); + if (defaultIsChunked) + return new MemPoolChunked (label, obj_size); else - return new MemMalloc (label, obj_size); + return new MemPoolMalloc (label, obj_size); } void @@ -420,40 +142,6 @@ defaultIsChunked = aBool; } -/* - * warning: we do not clean this entry from Pools assuming destruction - * is used at the end of the program only - */ -MemPool::~MemPool() -{ - MemChunk *chunk, *fchunk; - MemImplementingAllocator *find_pool, *prev_pool; - - flushMetersFull(); - clean(0); - assert(inuse == 0 && "While trying to destroy pool"); - - chunk = Chunks; - while ( (fchunk = chunk) != NULL) { - chunk = chunk->next; - delete fchunk; - } - /* TODO we should be doing something about the original Chunks pointer here. */ - - assert(MemPools::GetInstance().pools != NULL && "Called MemPool::~MemPool, but no pool exists!"); - - /* Pool clean, remove it from List and free */ - for (find_pool = MemPools::GetInstance().pools, prev_pool = NULL; (find_pool && this != find_pool); find_pool = find_pool->next) - prev_pool = find_pool; - assert(find_pool != NULL && "pool to destroy not found"); - - if (prev_pool) - prev_pool->next = next; - else - MemPools::GetInstance().pools = next; - --MemPools::GetInstance().poolCount; -} - char const * MemAllocator::objectType() const { @@ -473,24 +161,26 @@ calls = free_calls; if (calls) { - getMeter().gb_freed.count += calls; - memMeterDel(getMeter().inuse, calls); - memMeterAdd(getMeter().idle, calls); + meter.gb_freed.count += calls; free_calls = 0; } calls = alloc_calls; if (calls) { - meter.gb_saved.count += calls; - memMeterAdd(meter.inuse, calls); - memMeterDel(meter.idle, calls); + meter.gb_allocated.count += calls; alloc_calls = 0; } + calls = saved_calls; + if (calls) { + meter.gb_saved.count += calls; + saved_calls = 0; + } } void MemImplementingAllocator::flushMetersFull() { flushMeters(); + getMeter().gb_allocated.bytes = getMeter().gb_allocated.count * obj_size; getMeter().gb_saved.bytes = getMeter().gb_saved.count * obj_size; getMeter().gb_freed.bytes = getMeter().gb_freed.count * obj_size; } @@ -501,11 +191,21 @@ alloc.level = 0; inuse.level = 0; idle.level = 0; + gb_allocated.count = 0; + gb_allocated.bytes = 0; + gb_oallocated.count = 0; + gb_oallocated.bytes = 0; gb_saved.count = 0; gb_saved.bytes = 0; gb_freed.count = 0; gb_freed.bytes = 0; } + +MemPoolMeter::MemPoolMeter() +{ + flush(); +} + /* * Updates all pool counters, and recreates TheMeter totals from all pools */ @@ -523,8 +223,10 @@ memMeterAdd(TheMeter.alloc, pool->getMeter().alloc.level * pool->obj_size); memMeterAdd(TheMeter.inuse, pool->getMeter().inuse.level * pool->obj_size); memMeterAdd(TheMeter.idle, pool->getMeter().idle.level * pool->obj_size); + TheMeter.gb_allocated.count += pool->getMeter().gb_allocated.count; TheMeter.gb_saved.count += pool->getMeter().gb_saved.count; TheMeter.gb_freed.count += pool->getMeter().gb_freed.count; + TheMeter.gb_allocated.bytes += pool->getMeter().gb_allocated.bytes; TheMeter.gb_saved.bytes += pool->getMeter().gb_saved.bytes; TheMeter.gb_freed.bytes += pool->getMeter().gb_freed.bytes; } @@ -532,20 +234,6 @@ } void * -MemMalloc::allocate() -{ - inuse++; - return xcalloc(1, obj_size); -} - -void -MemMalloc::deallocate(void *obj) -{ - inuse--; - xfree(obj); -} - -void * MemImplementingAllocator::alloc() { if (++alloc_calls == FLUSH_LIMIT) @@ -559,127 +247,10 @@ { assert(obj != NULL); (void) VALGRIND_CHECK_MEM_IS_ADDRESSABLE(obj, obj_size); - deallocate(obj); + deallocate(obj, MemPools::GetInstance().mem_idle_limit == 0); ++free_calls; } -int -MemPool::getInUseCount() -{ - return inuse; -} - -void * -MemPool::allocate() -{ - void *p = get(); - assert(idle); - --idle; - ++inuse; - return p; -} - -void -MemPool::deallocate(void *obj) -{ - push(obj); - assert(inuse); - --inuse; - ++idle; -} - -void -MemPool::convertFreeCacheToChunkFreeCache() -{ - void *Free; - /* - * OK, so we have to go through all the global freeCache and find the Chunk - * any given Free belongs to, and stuff it into that Chunk's freelist - */ - - while ((Free = freeCache) != NULL) { - MemChunk *chunk = NULL; - chunk = const_cast(*allChunks.find(Free, memCompObjChunks)); - assert(splayLastResult == 0); - assert(chunk->inuse_count > 0); - chunk->inuse_count--; - (void) VALGRIND_MAKE_MEM_DEFINED(Free, sizeof(void *)); - freeCache = *(void **)Free; /* remove from global cache */ - *(void **)Free = chunk->freeList; /* stuff into chunks freelist */ - (void) VALGRIND_MAKE_MEM_NOACCESS(Free, sizeof(void *)); - chunk->freeList = Free; - chunk->lastref = squid_curtime; - } - -} - -/* removes empty Chunks from pool */ -void -MemPool::clean(time_t maxage) -{ - MemChunk *chunk, *freechunk, *listTail; - time_t age; - - if (!this) - return; - if (!Chunks) - return; - - flushMetersFull(); - convertFreeCacheToChunkFreeCache(); - /* Now we have all chunks in this pool cleared up, all free items returned to their home */ - /* We start now checking all chunks to see if we can release any */ - /* We start from Chunks->next, so first chunk is not released */ - /* Recreate nextFreeChunk list from scratch */ - - chunk = Chunks; - while ((freechunk = chunk->next) != NULL) { - age = squid_curtime - freechunk->lastref; - freechunk->nextFreeChunk = NULL; - if (freechunk->inuse_count == 0) - if (age >= maxage) { - chunk->next = freechunk->next; - delete freechunk; - freechunk = NULL; - } - if (chunk->next == NULL) - break; - chunk = chunk->next; - } - - /* Recreate nextFreeChunk list from scratch */ - /* Populate nextFreeChunk list in order of "most filled chunk first" */ - /* in case of equal fill, put chunk in lower ram first */ - /* First (create time) chunk is always on top, no matter how full */ - - chunk = Chunks; - nextFreeChunk = chunk; - chunk->nextFreeChunk = NULL; - - while (chunk->next) { - chunk->next->nextFreeChunk = NULL; - if (chunk->next->inuse_count < chunk_capacity) { - listTail = nextFreeChunk; - while (listTail->nextFreeChunk) { - if (chunk->next->inuse_count > listTail->nextFreeChunk->inuse_count) - break; - if ((chunk->next->inuse_count == listTail->nextFreeChunk->inuse_count) && - (chunk->next->objCache < listTail->nextFreeChunk->objCache)) - break; - listTail = listTail->nextFreeChunk; - } - chunk->next->nextFreeChunk = listTail->nextFreeChunk; - listTail->nextFreeChunk = chunk->next; - } - chunk = chunk->next; - } - /* We started from 2nd chunk. If first chunk is full, remove it */ - if (nextFreeChunk->inuse_count == chunk_capacity) - nextFreeChunk = nextFreeChunk->nextFreeChunk; - - return; -} - /* * Returns all cached frees to their home chunks * If chunks unreferenced age is over, destroys Idle chunk @@ -692,14 +263,16 @@ void MemPools::clean(time_t maxage) { - MemImplementingAllocator *pool; - MemPoolIterator *iter; + flushMeters(); + if (mem_idle_limit < 0) // no limit to enforce + return; int shift = 1; - flushMeters(); if (TheMeter.idle.level > mem_idle_limit) maxage = shift = 0; + MemImplementingAllocator *pool; + MemPoolIterator *iter; iter = memPoolIterate(); while ((pool = memPoolIterateNext(iter))) if (pool->idleTrigger(shift)) @@ -707,96 +280,10 @@ memPoolIterateDone(&iter); } -bool -MemPool::idleTrigger(int shift) const -{ - return getMeter().idle.level > (chunk_capacity << shift); -} - /* Persistent Pool stats. for GlobalStats accumulation */ static MemPoolStats pp_stats; /* - * Update MemPoolStats struct for single pool - */ -int -MemPool::getStats(MemPoolStats * stats) -{ - MemChunk *chunk; - int chunks_free = 0; - int chunks_partial = 0; - - if (stats != &pp_stats) /* need skip memset for GlobalStats accumulation */ - /* XXX Fixme */ - memset(stats, 0, sizeof(MemPoolStats)); - - clean((time_t) 555555); /* don't want to get chunks released before reporting */ - - stats->pool = this; - stats->label = objectType(); - stats->meter = &getMeter(); - stats->obj_size = obj_size; - stats->chunk_capacity = chunk_capacity; - - /* gather stats for each Chunk */ - chunk = Chunks; - while (chunk) { - if (chunk->inuse_count == 0) - chunks_free++; - else if (chunk->inuse_count < chunk_capacity) - chunks_partial++; - chunk = chunk->next; - } - - stats->chunks_alloc += chunkCount; - stats->chunks_inuse += chunkCount - chunks_free; - stats->chunks_partial += chunks_partial; - stats->chunks_free += chunks_free; - - stats->items_alloc += getMeter().alloc.level; - stats->items_inuse += getMeter().inuse.level; - stats->items_idle += getMeter().idle.level; - - stats->overhead += sizeof(MemPool) + chunkCount * sizeof(MemChunk) + strlen(objectType()) + 1; - - return getMeter().inuse.level; -} - -/* TODO extract common logic to MemAllocate */ -int -MemMalloc::getStats(MemPoolStats * stats) -{ - if (stats != &pp_stats) /* need skip memset for GlobalStats accumulation */ - /* XXX Fixme */ - memset(stats, 0, sizeof(MemPoolStats)); - - stats->pool = this; - stats->label = objectType(); - stats->meter = &getMeter(); - stats->obj_size = obj_size; - stats->chunk_capacity = 0; - - stats->chunks_alloc += 0; - stats->chunks_inuse += 0; - stats->chunks_partial += 0; - stats->chunks_free += 0; - - stats->items_alloc += getMeter().alloc.level; - stats->items_inuse += getMeter().inuse.level; - stats->items_idle += getMeter().idle.level; - - stats->overhead += sizeof(MemMalloc) + strlen(objectType()) + 1; - - return getMeter().inuse.level; -} - -int -MemMalloc::getInUseCount() -{ - return inuse; -} - -/* * Totals statistics is returned */ int @@ -814,7 +301,7 @@ /* gather all stats for Totals */ iter = memPoolIterate(); while ((pool = memPoolIterateNext(iter))) { - if (pool->getStats(&pp_stats) > 0) + if (pool->getStats(&pp_stats, 1) > 0) pools_inuse++; } memPoolIterateDone(&iter); @@ -833,7 +320,7 @@ stats->tot_items_inuse = pp_stats.items_inuse; stats->tot_items_idle = pp_stats.items_idle; - stats->tot_overhead += pp_stats.overhead + MemPools::GetInstance().poolCount * sizeof(MemPool *); + stats->tot_overhead += pp_stats.overhead + MemPools::GetInstance().poolCount * sizeof(MemAllocator *); stats->mem_idle_limit = MemPools::GetInstance().mem_idle_limit; return pools_inuse; @@ -848,19 +335,6 @@ return ((s + sizeof(void*) - 1) / sizeof(void*)) * sizeof(void*); } -MemMalloc::MemMalloc(char const *aLabel, size_t aSize) : MemImplementingAllocator(aLabel, aSize) { inuse = 0; } - -bool -MemMalloc::idleTrigger(int shift) const -{ - return false; -} - -void -MemMalloc::clean(time_t maxage) -{ -} - int memPoolInUseCount(MemAllocator * pool) { @@ -935,8 +409,39 @@ next(NULL), alloc_calls(0), free_calls(0), + saved_calls(0), obj_size(RoundedSize(aSize)) { + memPID = ++Pool_id_counter; + + MemImplementingAllocator *last_pool; + + assert(aLabel != NULL && aSize); + /* Append as Last */ + for (last_pool = MemPools::GetInstance().pools; last_pool && last_pool->next;) + last_pool = last_pool->next; + if (last_pool) + last_pool->next = this; + else + MemPools::GetInstance().pools = this; +} + +MemImplementingAllocator::~MemImplementingAllocator() +{ + MemImplementingAllocator *find_pool, *prev_pool; + + assert(MemPools::GetInstance().pools != NULL && "Called MemImplementingAllocator::~MemImplementingAllocator, but no pool exists!"); + + /* Pool clean, remove it from List and free */ + for (find_pool = MemPools::GetInstance().pools, prev_pool = NULL; (find_pool && this != find_pool); find_pool = find_pool->next) + prev_pool = find_pool; + assert(find_pool != NULL && "pool to destroy not found"); + + if (prev_pool) + prev_pool->next = next; + else + MemPools::GetInstance().pools = next; + --MemPools::GetInstance().poolCount; } void diff -u -r -N squid-3.1.3/lib/MemPoolChunked.cc squid-3.1.4/lib/MemPoolChunked.cc --- squid-3.1.3/lib/MemPoolChunked.cc 1970-01-01 12:00:00.000000000 +1200 +++ squid-3.1.4/lib/MemPoolChunked.cc 2010-05-31 01:21:11.000000000 +1200 @@ -0,0 +1,501 @@ + +/* + * $Id$ + * + * DEBUG: section 63 Low Level Memory Pool Management + * AUTHOR: Alex Rousskov, Andres Kroonmaa, Robert Collins + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * the Regents of the University of California. Please see the + * COPYRIGHT file for full details. Squid incorporates software + * developed and/or copyrighted by other sources. Please see the + * CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +/* + * Old way: + * xmalloc each item separately, upon free stack into idle pool array. + * each item is individually malloc()ed from system, imposing libmalloc + * overhead, and additionally we add our overhead of pointer size per item + * as we keep a list of pointer to free items. + * + * Chunking: + * xmalloc Chunk that fits at least MEM_MIN_FREE (32) items in an array, but + * limit Chunk size to MEM_CHUNK_MAX_SIZE (256K). Chunk size is rounded up to + * MEM_PAGE_SIZE (4K), trying to have chunks in multiples of VM_PAGE size. + * Minimum Chunk size is MEM_CHUNK_SIZE (16K). + * A number of items fits into a single chunk, depending on item size. + * Maximum number of items per chunk is limited to MEM_MAX_FREE (65535). + * + * We populate Chunk with a linkedlist, each node at first word of item, + * and pointing at next free item. Chunk->FreeList is pointing at first + * free node. Thus we stuff free housekeeping into the Chunk itself, and + * omit pointer overhead per item. + * + * Chunks are created on demand, and new chunks are inserted into linklist + * of chunks so that Chunks with smaller pointer value are placed closer + * to the linklist head. Head is a hotspot, servicing most of requests, so + * slow sorting occurs and Chunks in highest memory tend to become idle + * and freeable. + * + * event is registered that runs every 15 secs and checks reference time + * of each idle chunk. If a chunk is not referenced for 15 secs, it is + * released. + * + * [If mem_idle_limit is exceeded with pools, every chunk that becomes + * idle is immediately considered for release, unless this is the only + * chunk with free items in it.] (not implemented) + * + * In cachemgr output, there are new columns for chunking. Special item, + * Frag, is shown to estimate approximately fragmentation of chunked + * pools. Fragmentation is calculated by taking amount of items in use, + * calculating needed amount of chunks to fit all, and then comparing to + * actual amount of chunks in use. Frag number, in percent, is showing + * how many percent of chunks are in use excessively. 100% meaning that + * twice the needed amount of chunks are in use. + * "part" item shows number of chunks partially filled. This shows how + * badly fragmentation is spread across all chunks. + * + * Andres Kroonmaa. + * Copyright (c) 2003, Robert Collins + */ + +#include "config.h" +#if HAVE_ASSERT_H +#include +#endif + +#include "MemPoolChunked.h" + +#define MEM_MAX_MMAP_CHUNKS 2048 + +#if HAVE_STRING_H +#include +#endif + +/* + * XXX This is a boundary violation between lib and src.. would be good + * if it could be solved otherwise, but left for now. + */ +extern time_t squid_curtime; + +/* local prototypes */ +static int memCompChunks(MemChunk * const &, MemChunk * const &); +static int memCompObjChunks(void * const &, MemChunk * const &); + +/* Compare chunks */ +static int +memCompChunks(MemChunk * const &chunkA, MemChunk * const &chunkB) +{ + if (chunkA->objCache > chunkB->objCache) + return 1; + else if (chunkA->objCache < chunkB->objCache) + return -1; + else + return 0; +} + +/* Compare object to chunk */ +static int +memCompObjChunks(void *const &obj, MemChunk * const &chunk) +{ + /* object is lower in memory than the chunks arena */ + if (obj < chunk->objCache) + return -1; + /* object is within the pool */ + if (obj < (void *) ((char *) chunk->objCache + chunk->pool->chunk_size)) + return 0; + /* object is above the pool */ + return 1; +} + +MemChunk::MemChunk(MemPoolChunked *aPool) +{ + /* should have a pool for this too - + * note that this requres: + * allocate one chunk for the pool of chunks's first chunk + * allocate a chunk from that pool + * move the contents of one chunk into the other + * free the first chunk. + */ + inuse_count = 0; + next = NULL; + pool = aPool; + + objCache = xcalloc(1, pool->chunk_size); + freeList = objCache; + void **Free = (void **)freeList; + + for (int i = 1; i < pool->chunk_capacity; i++) { + *Free = (void *) ((char *) Free + pool->obj_size); + void **nextFree = (void **)*Free; + (void) VALGRIND_MAKE_MEM_NOACCESS(Free, pool->obj_size); + Free = nextFree; + } + nextFreeChunk = pool->nextFreeChunk; + pool->nextFreeChunk = this; + + memMeterAdd(pool->getMeter().alloc, pool->chunk_capacity); + memMeterAdd(pool->getMeter().idle, pool->chunk_capacity); + pool->chunkCount++; + lastref = squid_curtime; + pool->allChunks.insert(this, memCompChunks); +} + +MemPoolChunked::MemPoolChunked(const char *aLabel, size_t aSize) : MemImplementingAllocator(aLabel, aSize) +{ + chunk_size = 0; + chunk_capacity = 0; + chunkCount = 0; + freeCache = 0; + nextFreeChunk = 0; + Chunks = 0; + next = 0; + + setChunkSize(MEM_CHUNK_SIZE); + +#if HAVE_MALLOPT && M_MMAP_MAX + mallopt(M_MMAP_MAX, MEM_MAX_MMAP_CHUNKS); +#endif +} + +MemChunk::~MemChunk() +{ + memMeterDel(pool->getMeter().alloc, pool->chunk_capacity); + memMeterDel(pool->getMeter().idle, pool->chunk_capacity); + pool->chunkCount--; + pool->allChunks.remove(this, memCompChunks); + xfree(objCache); +} + +void +MemPoolChunked::push(void *obj) +{ + void **Free; + /* XXX We should figure out a sane way of avoiding having to clear + * all buffers. For example data buffers such as used by MemBuf do + * not really need to be cleared.. There was a condition based on + * the object size here, but such condition is not safe. + */ + if (doZeroOnPush) + memset(obj, 0, obj_size); + Free = (void **)obj; + *Free = freeCache; + freeCache = obj; + (void) VALGRIND_MAKE_MEM_NOACCESS(obj, obj_size); +} + +/* + * Find a chunk with a free item. + * Create new chunk on demand if no chunk with frees found. + * Insert new chunk in front of lowest ram chunk, making it preferred in future, + * and resulting slow compaction towards lowest ram area. + */ +void * +MemPoolChunked::get() +{ + void **Free; + + saved_calls++; + + /* first, try cache */ + if (freeCache) { + Free = (void **)freeCache; + (void) VALGRIND_MAKE_MEM_DEFINED(Free, obj_size); + freeCache = *Free; + *Free = NULL; + return Free; + } + /* then try perchunk freelist chain */ + if (nextFreeChunk == NULL) { + /* no chunk with frees, so create new one */ + saved_calls--; // compensate for the ++ above + createChunk(); + } + /* now we have some in perchunk freelist chain */ + MemChunk *chunk = nextFreeChunk; + + Free = (void **)chunk->freeList; + chunk->freeList = *Free; + *Free = NULL; + chunk->inuse_count++; + chunk->lastref = squid_curtime; + + if (chunk->freeList == NULL) { + /* last free in this chunk, so remove us from perchunk freelist chain */ + nextFreeChunk = chunk->nextFreeChunk; + } + (void) VALGRIND_MAKE_MEM_DEFINED(Free, obj_size); + return Free; +} + +/* just create a new chunk and place it into a good spot in the chunk chain */ +void +MemPoolChunked::createChunk() +{ + MemChunk *chunk, *newChunk; + + newChunk = new MemChunk(this); + + chunk = Chunks; + if (chunk == NULL) { /* first chunk in pool */ + Chunks = newChunk; + return; + } + if (newChunk->objCache < chunk->objCache) { + /* we are lowest ram chunk, insert as first chunk */ + newChunk->next = chunk; + Chunks = newChunk; + return; + } + while (chunk->next) { + if (newChunk->objCache < chunk->next->objCache) { + /* new chunk is in lower ram, insert here */ + newChunk->next = chunk->next; + chunk->next = newChunk; + return; + } + chunk = chunk->next; + } + /* we are the worst chunk in chain, add as last */ + chunk->next = newChunk; +} + +void +MemPoolChunked::setChunkSize(size_t chunksize) +{ + int cap; + size_t csize = chunksize; + + if (Chunks) /* unsafe to tamper */ + return; + + csize = ((csize + MEM_PAGE_SIZE - 1) / MEM_PAGE_SIZE) * MEM_PAGE_SIZE; /* round up to page size */ + cap = csize / obj_size; + + if (cap < MEM_MIN_FREE) + cap = MEM_MIN_FREE; + if (cap * obj_size > MEM_CHUNK_MAX_SIZE) + cap = MEM_CHUNK_MAX_SIZE / obj_size; + if (cap > MEM_MAX_FREE) + cap = MEM_MAX_FREE; + if (cap < 1) + cap = 1; + + csize = cap * obj_size; + csize = ((csize + MEM_PAGE_SIZE - 1) / MEM_PAGE_SIZE) * MEM_PAGE_SIZE; /* round up to page size */ + cap = csize / obj_size; + + chunk_capacity = cap; + chunk_size = csize; +} + +/* + * warning: we do not clean this entry from Pools assuming destruction + * is used at the end of the program only + */ +MemPoolChunked::~MemPoolChunked() +{ + MemChunk *chunk, *fchunk; + + flushMetersFull(); + clean(0); + assert(meter.inuse.level == 0 && "While trying to destroy pool"); + + chunk = Chunks; + while ( (fchunk = chunk) != NULL) { + chunk = chunk->next; + delete fchunk; + } + /* TODO we should be doing something about the original Chunks pointer here. */ + +} + +int +MemPoolChunked::getInUseCount() +{ + return meter.inuse.level; +} + +void * +MemPoolChunked::allocate() +{ + void *p = get(); + assert(meter.idle.level > 0); + memMeterDec(meter.idle); + memMeterInc(meter.inuse); + return p; +} + +void +MemPoolChunked::deallocate(void *obj, bool aggressive) +{ + push(obj); + assert(meter.inuse.level > 0); + memMeterDec(meter.inuse); + memMeterInc(meter.idle); +} + +void +MemPoolChunked::convertFreeCacheToChunkFreeCache() +{ + void *Free; + /* + * OK, so we have to go through all the global freeCache and find the Chunk + * any given Free belongs to, and stuff it into that Chunk's freelist + */ + + while ((Free = freeCache) != NULL) { + MemChunk *chunk = NULL; + chunk = const_cast(*allChunks.find(Free, memCompObjChunks)); + assert(splayLastResult == 0); + assert(chunk->inuse_count > 0); + chunk->inuse_count--; + (void) VALGRIND_MAKE_MEM_DEFINED(Free, sizeof(void *)); + freeCache = *(void **)Free; /* remove from global cache */ + *(void **)Free = chunk->freeList; /* stuff into chunks freelist */ + (void) VALGRIND_MAKE_MEM_NOACCESS(Free, sizeof(void *)); + chunk->freeList = Free; + chunk->lastref = squid_curtime; + } + +} + +/* removes empty Chunks from pool */ +void +MemPoolChunked::clean(time_t maxage) +{ + MemChunk *chunk, *freechunk, *listTail; + time_t age; + + if (!this) + return; + if (!Chunks) + return; + + flushMetersFull(); + convertFreeCacheToChunkFreeCache(); + /* Now we have all chunks in this pool cleared up, all free items returned to their home */ + /* We start now checking all chunks to see if we can release any */ + /* We start from Chunks->next, so first chunk is not released */ + /* Recreate nextFreeChunk list from scratch */ + + chunk = Chunks; + while ((freechunk = chunk->next) != NULL) { + age = squid_curtime - freechunk->lastref; + freechunk->nextFreeChunk = NULL; + if (freechunk->inuse_count == 0) + if (age >= maxage) { + chunk->next = freechunk->next; + delete freechunk; + freechunk = NULL; + } + if (chunk->next == NULL) + break; + chunk = chunk->next; + } + + /* Recreate nextFreeChunk list from scratch */ + /* Populate nextFreeChunk list in order of "most filled chunk first" */ + /* in case of equal fill, put chunk in lower ram first */ + /* First (create time) chunk is always on top, no matter how full */ + + chunk = Chunks; + nextFreeChunk = chunk; + chunk->nextFreeChunk = NULL; + + while (chunk->next) { + chunk->next->nextFreeChunk = NULL; + if (chunk->next->inuse_count < chunk_capacity) { + listTail = nextFreeChunk; + while (listTail->nextFreeChunk) { + if (chunk->next->inuse_count > listTail->nextFreeChunk->inuse_count) + break; + if ((chunk->next->inuse_count == listTail->nextFreeChunk->inuse_count) && + (chunk->next->objCache < listTail->nextFreeChunk->objCache)) + break; + listTail = listTail->nextFreeChunk; + } + chunk->next->nextFreeChunk = listTail->nextFreeChunk; + listTail->nextFreeChunk = chunk->next; + } + chunk = chunk->next; + } + /* We started from 2nd chunk. If first chunk is full, remove it */ + if (nextFreeChunk->inuse_count == chunk_capacity) + nextFreeChunk = nextFreeChunk->nextFreeChunk; + + return; +} + +bool +MemPoolChunked::idleTrigger(int shift) const +{ + return meter.idle.level > (chunk_capacity << shift); +} + +/* + * Update MemPoolStats struct for single pool + */ +int +MemPoolChunked::getStats(MemPoolStats * stats, int accumulate) +{ + MemChunk *chunk; + int chunks_free = 0; + int chunks_partial = 0; + + if (!accumulate) /* need skip memset for GlobalStats accumulation */ + memset(stats, 0, sizeof(MemPoolStats)); + + clean((time_t) 555555); /* don't want to get chunks released before reporting */ + + stats->pool = this; + stats->label = objectType(); + stats->meter = &meter; + stats->obj_size = obj_size; + stats->chunk_capacity = chunk_capacity; + + /* gather stats for each Chunk */ + chunk = Chunks; + while (chunk) { + if (chunk->inuse_count == 0) + chunks_free++; + else if (chunk->inuse_count < chunk_capacity) + chunks_partial++; + chunk = chunk->next; + } + + stats->chunks_alloc += chunkCount; + stats->chunks_inuse += chunkCount - chunks_free; + stats->chunks_partial += chunks_partial; + stats->chunks_free += chunks_free; + + stats->items_alloc += meter.alloc.level; + stats->items_inuse += meter.inuse.level; + stats->items_idle += meter.idle.level; + + stats->overhead += sizeof(MemPoolChunked) + chunkCount * sizeof(MemChunk) + strlen(objectType()) + 1; + + return meter.inuse.level; +} diff -u -r -N squid-3.1.3/lib/MemPoolMalloc.cc squid-3.1.4/lib/MemPoolMalloc.cc --- squid-3.1.3/lib/MemPoolMalloc.cc 1970-01-01 12:00:00.000000000 +1200 +++ squid-3.1.4/lib/MemPoolMalloc.cc 2010-05-31 01:21:11.000000000 +1200 @@ -0,0 +1,142 @@ + +/* + * $Id$ + * + * DEBUG: section 63 Low Level Memory Pool Management + * AUTHOR: Alex Rousskov, Andres Kroonmaa, Robert Collins, Henrik Nordstrom + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * the Regents of the University of California. Please see the + * COPYRIGHT file for full details. Squid incorporates software + * developed and/or copyrighted by other sources. Please see the + * CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + + +#include "config.h" +#if HAVE_ASSERT_H +#include +#endif + +#include "MemPoolMalloc.h" + +#if HAVE_STRING_H +#include +#endif + +/* + * XXX This is a boundary violation between lib and src.. would be good + * if it could be solved otherwise, but left for now. + */ +extern time_t squid_curtime; + +void * +MemPoolMalloc::allocate() +{ + void *obj = freelist.pop(); + if (obj) { + memMeterDec(meter.idle); + saved_calls++; + } else { + obj = xcalloc(1, obj_size); + memMeterInc(meter.alloc); + } + memMeterInc(meter.inuse); + return obj; +} + +void +MemPoolMalloc::deallocate(void *obj, bool aggressive) +{ + memMeterDec(meter.inuse); + if (aggressive) { + xfree(obj); + memMeterDec(meter.alloc); + } else { + if (doZeroOnPush) + memset(obj, 0, obj_size); + memMeterInc(meter.idle); + freelist.push_back(obj); + } +} + +/* TODO extract common logic to MemAllocate */ +int +MemPoolMalloc::getStats(MemPoolStats * stats, int accumulate) +{ + if (!accumulate) /* need skip memset for GlobalStats accumulation */ + memset(stats, 0, sizeof(MemPoolStats)); + + stats->pool = this; + stats->label = objectType(); + stats->meter = &meter; + stats->obj_size = obj_size; + stats->chunk_capacity = 0; + + stats->chunks_alloc += 0; + stats->chunks_inuse += 0; + stats->chunks_partial += 0; + stats->chunks_free += 0; + + stats->items_alloc += meter.alloc.level; + stats->items_inuse += meter.inuse.level; + stats->items_idle += meter.idle.level; + + stats->overhead += sizeof(MemPoolMalloc) + strlen(objectType()) + 1; + + return meter.inuse.level; +} + +int +MemPoolMalloc::getInUseCount() +{ + return meter.inuse.level; +} + +MemPoolMalloc::MemPoolMalloc(char const *aLabel, size_t aSize) : MemImplementingAllocator(aLabel, aSize) +{ +} + +MemPoolMalloc::~MemPoolMalloc() +{ + assert(meter.inuse.level == 0 && "While trying to destroy pool"); + clean(0); +} + +bool +MemPoolMalloc::idleTrigger(int shift) const +{ + return freelist.count >> (shift ? 8 : 0); +} + +void +MemPoolMalloc::clean(time_t maxage) +{ + while (void *obj = freelist.pop()) { + memMeterDec(meter.idle); + memMeterDec(meter.alloc); + xfree(obj); + } +} + diff -u -r -N squid-3.1.3/RELEASENOTES.html squid-3.1.4/RELEASENOTES.html --- squid-3.1.3/RELEASENOTES.html 2010-05-02 23:04:50.000000000 +1200 +++ squid-3.1.4/RELEASENOTES.html 2010-05-31 01:32:48.000000000 +1200 @@ -2,10 +2,10 @@ - Squid 3.1.3 release notes + Squid 3.1.4 release notes -

Squid 3.1.3 release notes

+

Squid 3.1.4 release notes

Squid Developers

@@ -70,7 +70,7 @@

1. Notice

-

The Squid Team are pleased to announce the release of Squid-3.1.3.

+

The Squid Team are pleased to announce the release of Squid-3.1.4.

This new release is available for download from http://www.squid-cache.org/Versions/v3/3.1/ or the mirrors.

@@ -1426,6 +1426,11 @@
maximum_object_size_in_memory

Default size limit increased to 512KB.

+
memory_pools_limit
+

Memory limits have been revised and corrected from 3.1.4 onwards.

+

Please check and update your squid.conf to use the text none for no limit instead of the old 0 (zero).

+

All users upgrading need to be aware that from Squid-3.3 setting this option to 0 (zero) will mean zero bytes of memory get pooled.

+
negative_ttl

New default of 0 seconds. To prevent negative-caching of failure messages unless explicitly permitted by the message generating web server.

@@ -1562,11 +1567,6 @@

Disable error page localization for visitors.

error_directory option is required if this option is used.

-
--disable-caps
-

Build without libcap support. The default is to auto-detect system capabilities -and enable support when possible.

-

NOTE: Disabling this or building without libcap support will break TPROXY support.

-
--disable-ipv6

Build without IPv6 support. The default is to auto-detect system capabilities and build with IPv6 when possible.

@@ -1582,7 +1582,7 @@ to the squid developers before doing so.

--disable-translation
-

Prevent Squid generating localized error page templates and manuals. +

Prevent Squid generating localized error page templates and manuals when built. Which is usually tried, but may not be needed.

This is an optimization for building fast when localization is not needed or localization tools are not available.

@@ -1624,6 +1624,11 @@

Absolute path to po2html executable. Default is to automatically detect the binary.

+
--without-libcap
+

Build without libcap support. The default is to auto-detect system capabilities +and enable support when possible.

+

NOTE: Disabling this or building without libcap support will break TPROXY support.

+

@@ -1677,6 +1682,9 @@
--disable-carp

Removed. CARP is required by several peering algoithms. Disabling is not useful.

+ +
--disable-mempools
+

Replaced by memory_pools squid.conf option.

diff -u -r -N squid-3.1.3/src/access_log.cc squid-3.1.4/src/access_log.cc --- squid-3.1.3/src/access_log.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/access_log.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1807,7 +1807,7 @@ user2 = accessLogFormatName(al->cache.rfc931); - logfilePrintf(logfile, "%s %s %s [%s] \"%s %s HTTP/%d.%d\" %d %"PRId64" %s:%s", + logfilePrintf(logfile, "%s %s %s [%s] \"%s %s HTTP/%d.%d\" %d %"PRId64" %s:%s%s", client, user2 ? user2 : dash_str, user1 ? user1 : dash_str, @@ -1818,7 +1818,8 @@ al->http.code, al->cache.replySize, log_tags[al->cache.code], - hier_strings[al->hier.code]); + hier_strings[al->hier.code], + (Config.onoff.log_mime_hdrs?"":"\n")); safe_free(user1); @@ -1830,10 +1831,7 @@ logfilePrintf(logfile, " [%s] [%s]\n", ereq, erep); safe_free(ereq); safe_free(erep); - } else { - logfilePrintf(logfile, "\n"); } - } #if ICAP_CLIENT @@ -2440,4 +2438,3 @@ return 0; } - diff -u -r -N squid-3.1.3/src/adaptation/Config.cc squid-3.1.4/src/adaptation/Config.cc --- squid-3.1.3/src/adaptation/Config.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/Config.cc 2010-05-31 01:21:11.000000000 +1200 @@ -62,6 +62,11 @@ void Adaptation::Config::freeService() { + FreeAccess(); + FreeServiceGroups(); + + DetachServices(); + while (!serviceConfigs.empty()) { delete serviceConfigs.back(); serviceConfigs.pop_back(); @@ -208,14 +213,5 @@ // with global arrays shared by those individual configs Adaptation::Config::~Config() { - FreeAccess(); - FreeServiceGroups(); - - // invalidate each service so that it can be deleted when refcount=0 - while (!AllServices().empty()) { - AllServices().back()->invalidate(); - AllServices().pop_back(); - } - freeService(); } diff -u -r -N squid-3.1.3/src/adaptation/ecap/Config.cc squid-3.1.4/src/adaptation/ecap/Config.cc --- squid-3.1.3/src/adaptation/ecap/Config.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/ecap/Config.cc 2010-05-31 01:21:12.000000000 +1200 @@ -1,11 +1,9 @@ - /* * $Id$ + * DEBUG: section 93 eCAP Interface */ - #include "squid.h" -#include #include "adaptation/ecap/Host.h" #include "adaptation/ecap/ServiceRep.h" #include "adaptation/ecap/Config.h" @@ -24,8 +22,8 @@ Adaptation::Ecap::Config::finalize() { Adaptation::Config::finalize(); - libecap::shared_ptr host(new Adaptation::Ecap::Host); - libecap::RegisterHost(host); + Host::Register(); + CheckUnusedAdapterServices(AllServices()); } Adaptation::ServicePointer diff -u -r -N squid-3.1.3/src/adaptation/ecap/Config.h squid-3.1.4/src/adaptation/ecap/Config.h --- squid-3.1.3/src/adaptation/ecap/Config.h 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/ecap/Config.h 2010-05-31 01:21:11.000000000 +1200 @@ -1,7 +1,6 @@ - /* * $Id$ - * + * DEBUG: section 93 eCAP Interface */ #ifndef SQUID_ECAP_CONFIG_H diff -u -r -N squid-3.1.3/src/adaptation/ecap/Host.cc squid-3.1.4/src/adaptation/ecap/Host.cc --- squid-3.1.3/src/adaptation/ecap/Host.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/ecap/Host.cc 2010-05-31 01:21:12.000000000 +1200 @@ -1,6 +1,10 @@ +/* + * DEBUG: section 93 eCAP Interface + */ #include "squid.h" #include #include +#include #include "TextException.h" #include "adaptation/ecap/ServiceRep.h" #include "adaptation/ecap/Host.h" @@ -12,9 +16,14 @@ const libecap::Name Adaptation::Ecap::protocolHtcp("Htcp", libecap::Name::NextId()); #endif +/// the host application (i.e., Squid) wrapper registered with libecap +static libecap::shared_ptr TheHost; + Adaptation::Ecap::Host::Host() { // assign our host-specific IDs to well-known names + // this code can run only once + libecap::headerReferer.assignHostId(HDR_REFERER); libecap::protocolHttp.assignHostId(PROTO_HTTP); @@ -47,28 +56,8 @@ void Adaptation::Ecap::Host::noteService(const libecap::weak_ptr &weak) { - // Many ecap_service lines may use the same service URI. Find each - // matching service rep, make sure it is an eCAP rep, - // and update it with the actual eCAP service. - int found = 0; - - libecap::shared_ptr shared(weak); - typedef Adaptation::Services::iterator SI; - for (SI i = Adaptation::AllServices().begin(); i != Adaptation::AllServices().end(); ++i) { - if ((*i)->cfg().uri == shared->uri().c_str()) { - ServiceRep *rep = dynamic_cast(i->getRaw()); - Must(rep); - rep->noteService(shared); - ++found; - } - } - - debugs(93,5, HERE << "Found " << found << " ecap_service configs for " << - shared->uri()); - if (!found) { - debugs(93,1, "Warning: ignoring loaded eCAP module service without " << - "a matching ecap_service configuration: " << shared->uri()); - } + Must(!weak.expired()); + RegisterAdapterService(weak.lock()); } static int @@ -104,3 +93,12 @@ if (debug) Debug::finishDebug(); } + +void +Adaptation::Ecap::Host::Register() +{ + if (!TheHost) { + TheHost.reset(new Adaptation::Ecap::Host); + libecap::RegisterHost(TheHost); + } +} diff -u -r -N squid-3.1.3/src/adaptation/ecap/Host.h squid-3.1.4/src/adaptation/ecap/Host.h --- squid-3.1.3/src/adaptation/ecap/Host.h 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/ecap/Host.h 2010-05-31 01:21:11.000000000 +1200 @@ -1,6 +1,6 @@ - /* * $Id$ + * DEBUG: section 93 eCAP Interface */ #ifndef SQUID_ECAP_HOST_H @@ -17,8 +17,6 @@ class Host : public libecap::host::Host { public: - Host(); - // About virtual std::string uri() const; // unique across all vendors virtual void describe(std::ostream &os) const; // free-format info @@ -29,6 +27,13 @@ // Logging virtual std::ostream *openDebug(libecap::LogVerbosity lv); virtual void closeDebug(std::ostream *debug); + + static void Register(); ///< register adaptation host + +private: + Host(); + Host (const Host&); ///< not implemented + Host& operator= (const Host&); ///< not implemented }; extern const libecap::Name protocolInternal; diff -u -r -N squid-3.1.3/src/adaptation/ecap/MessageRep.cc squid-3.1.4/src/adaptation/ecap/MessageRep.cc --- squid-3.1.3/src/adaptation/ecap/MessageRep.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/ecap/MessageRep.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,7 +1,6 @@ /* - * DEBUG: section XXX + * DEBUG: section 93 eCAP Interface */ - #include "squid.h" #include "HttpRequest.h" #include "HttpReply.h" @@ -38,7 +37,8 @@ const String value = squidId == HDR_OTHER ? theHeader.getByName(name.image().c_str()) : theHeader.getStrOrList(squidId); - return Value::FromTempString(value.termedBuf()); + return value.defined() ? + Value::FromTempString(value.termedBuf()) : Value(); } void diff -u -r -N squid-3.1.3/src/adaptation/ecap/MessageRep.h squid-3.1.4/src/adaptation/ecap/MessageRep.h --- squid-3.1.3/src/adaptation/ecap/MessageRep.h 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/ecap/MessageRep.h 2010-05-31 01:21:11.000000000 +1200 @@ -1,6 +1,6 @@ - /* * $Id$ + * DEBUG: section 93 eCAP Interface */ #ifndef SQUID__ECAP__MESSAGE_REP_H diff -u -r -N squid-3.1.3/src/adaptation/ecap/ServiceRep.cc squid-3.1.4/src/adaptation/ecap/ServiceRep.cc --- squid-3.1.3/src/adaptation/ecap/ServiceRep.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/ecap/ServiceRep.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,11 +1,19 @@ +/* + * DEBUG: section 93 eCAP Interface + */ #include "squid.h" +#include #include #include "TextException.h" #include "adaptation/ecap/ServiceRep.h" #include "adaptation/ecap/XactionRep.h" +// configured eCAP service wrappers +static std::list TheServices; + Adaptation::Ecap::ServiceRep::ServiceRep(const Adaptation::ServiceConfig &cfg): - /*AsyncJob("Adaptation::Ecap::ServiceRep"),*/ Adaptation::Service(cfg) + /*AsyncJob("Adaptation::Ecap::ServiceRep"),*/ Adaptation::Service(cfg), + isDetached(false) { } @@ -13,20 +21,6 @@ { } -void Adaptation::Ecap::ServiceRep::noteService(const AdapterService &s) -{ - Must(s != NULL); - theService = s; - debugs(93,7, HERE << "matched loaded and configured eCAP services: " << - s->uri() << ' ' << cfg().key << "\n"); -} - -void Adaptation::Ecap::ServiceRep::invalidate() -{ - theService->retire(); - theService.reset(); -} - void Adaptation::Ecap::ServiceRep::noteFailure() { assert(false); // XXX: should this be ICAP-specific? @@ -36,6 +30,7 @@ Adaptation::Ecap::ServiceRep::finalize() { Adaptation::Service::finalize(); + theService = FindAdapterService(cfg().uri); if (theService) { debugs(93,3, HERE << "starting eCAP service: " << theService->uri()); theService->start(); @@ -75,7 +70,92 @@ // returns a temporary string depicting service status, for debugging const char *Adaptation::Ecap::ServiceRep::status() const { - assert(false); // move generic stuff from ICAP to Adaptation - // add theService->status()? - return NULL; + // TODO: move generic stuff from eCAP and ICAP to Adaptation + static MemBuf buf; + + buf.reset(); + buf.append("[", 1); + + if (up()) + buf.append("up", 2); + else + buf.append("down", 4); + + if (detached()) + buf.append(",detached", 9); + + buf.append("]", 1); + buf.terminate(); + + return buf.content(); +} + +void Adaptation::Ecap::ServiceRep::detach() +{ + isDetached = true; +} + +bool Adaptation::Ecap::ServiceRep::detached() const +{ + return isDetached; +} + +Adaptation::Ecap::ServiceRep::AdapterService +Adaptation::Ecap::FindAdapterService(const String& serviceUri) +{ + typedef std::list::const_iterator ASCI; + for (ASCI s = TheServices.begin(); s != TheServices.end(); ++s) { + Must(*s); + if (serviceUri == (*s)->uri().c_str()) + return *s; + } + return ServiceRep::AdapterService(); +} + +void +Adaptation::Ecap::RegisterAdapterService(const Adaptation::Ecap::ServiceRep::AdapterService& adapterService) +{ + typedef std::list::iterator ASI; + for (ASI s = TheServices.begin(); s != TheServices.end(); ++s) { + Must(*s); + if (adapterService->uri() == (*s)->uri()) { + *s = adapterService; + debugs(93, 3, "updated eCAP module service: " << + adapterService->uri()); + return; + } + } + TheServices.push_back(adapterService); + debugs(93, 3, "registered eCAP module service: " << adapterService->uri()); +} + +void +Adaptation::Ecap::UnregisterAdapterService(const String& serviceUri) +{ + typedef std::list::iterator ASI; + for (ASI s = TheServices.begin(); s != TheServices.end(); ++s) { + if (serviceUri == (*s)->uri().c_str()) { + TheServices.erase(s); + debugs(93, 3, "unregistered eCAP module service: " << serviceUri); + return; + } + } + debugs(93, 3, "failed to unregister eCAP module service: " << serviceUri); +} + +void +Adaptation::Ecap::CheckUnusedAdapterServices(const Adaptation::Services& cfgs) +{ + typedef std::list::const_iterator ASCI; + for (ASCI loaded = TheServices.begin(); loaded != TheServices.end(); + ++loaded) { + bool found = false; + for (Services::const_iterator cfged = cfgs.begin(); + cfged != cfgs.end() && !found; ++cfged) { + found = (*cfged)->cfg().uri == (*loaded)->uri().c_str(); + } + if (!found) + debugs(93, 1, "Warning: loaded eCAP service has no matching " << + "ecap_service config option: " << (*loaded)->uri()); + } } diff -u -r -N squid-3.1.3/src/adaptation/ecap/ServiceRep.h squid-3.1.4/src/adaptation/ecap/ServiceRep.h --- squid-3.1.3/src/adaptation/ecap/ServiceRep.h 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/adaptation/ecap/ServiceRep.h 2010-05-31 01:21:11.000000000 +1200 @@ -1,6 +1,6 @@ - /* * $Id$ + * DEBUG: section 93 eCAP Interface */ #ifndef SQUID_ECAP_SERVICE_REP_H @@ -27,13 +27,9 @@ virtual ~ServiceRep(); typedef libecap::shared_ptr AdapterService; - void noteService(const AdapterService &s); virtual void finalize(); - // call when the service is no longer needed or valid - virtual void invalidate(); - virtual bool probed() const; virtual bool up() const; @@ -47,10 +43,24 @@ virtual const char *status() const; + virtual void detach(); + virtual bool detached() const; + private: AdapterService theService; // the actual adaptation service we represent + bool isDetached; }; +/// register loaded eCAP module service +extern void RegisterAdapterService(const ServiceRep::AdapterService& adapterService); +/// unregister loaded eCAP module service by service uri +extern void UnregisterAdapterService(const String& serviceUri); + +/// returns loaded eCAP module service by service uri +extern ServiceRep::AdapterService FindAdapterService(const String& serviceUri); + +/// check for loaded eCAP services without matching ecap_service in squid.conf +extern void CheckUnusedAdapterServices(const Services& services); } // namespace Ecap } // namespace Adaptation diff -u -r -N squid-3.1.3/src/adaptation/ecap/XactionRep.cc squid-3.1.4/src/adaptation/ecap/XactionRep.cc --- squid-3.1.3/src/adaptation/ecap/XactionRep.cc 2010-05-02 22:47:09.000000000 +1200 +++ squid-3.1.4/src/adaptation/ecap/XactionRep.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,3 +1,6 @@ +/* + * DEBUG: section 93 eCAP Interface + */ #include "squid.h" #include #include diff -u -r -N squid-3.1.3/src/adaptation/ecap/XactionRep.h squid-3.1.4/src/adaptation/ecap/XactionRep.h --- squid-3.1.3/src/adaptation/ecap/XactionRep.h 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/adaptation/ecap/XactionRep.h 2010-05-31 01:21:12.000000000 +1200 @@ -1,6 +1,6 @@ - /* * $Id$ + * DEBUG: section 93 eCAP Interface */ #ifndef SQUID_ECAP_XACTION_REP_H diff -u -r -N squid-3.1.3/src/adaptation/icap/Config.cc squid-3.1.4/src/adaptation/icap/Config.cc --- squid-3.1.3/src/adaptation/icap/Config.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/icap/Config.cc 2010-05-31 01:21:11.000000000 +1200 @@ -59,9 +59,7 @@ Adaptation::ServicePointer Adaptation::Icap::Config::createService(const Adaptation::ServiceConfig &cfg) { - Adaptation::Icap::ServiceRep::Pointer s = new Adaptation::Icap::ServiceRep(cfg); - s->setSelf(s); - return s.getRaw(); + return new Adaptation::Icap::ServiceRep(cfg); } time_t Adaptation::Icap::Config::connect_timeout(bool bypassable) const diff -u -r -N squid-3.1.3/src/adaptation/icap/Launcher.cc squid-3.1.4/src/adaptation/icap/Launcher.cc --- squid-3.1.3/src/adaptation/icap/Launcher.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/icap/Launcher.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section 93 ICAP (RFC 3507) Client + * DEBUG: section 93 ICAP (RFC 3507) Client */ #include "squid.h" diff -u -r -N squid-3.1.3/src/adaptation/icap/ServiceRep.cc squid-3.1.4/src/adaptation/icap/ServiceRep.cc --- squid-3.1.3/src/adaptation/icap/ServiceRep.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/icap/ServiceRep.cc 2010-05-31 01:21:11.000000000 +1200 @@ -19,28 +19,21 @@ AsyncJob("Adaptation::Icap::ServiceRep"), Adaptation::Service(svcCfg), theOptions(NULL), theOptionsFetcher(0), theLastUpdate(0), theSessionFailures(0), isSuspended(0), notifying(false), - updateScheduled(false), self(NULL), - wasAnnouncedUp(true) // do not announce an "up" service at startup + updateScheduled(false), + wasAnnouncedUp(true), // do not announce an "up" service at startup + isDetached(false) {} Adaptation::Icap::ServiceRep::~ServiceRep() { Must(!theOptionsFetcher); - changeOptions(0); -} - -void -Adaptation::Icap::ServiceRep::setSelf(Pointer &aSelf) -{ - assert(!self && aSelf != NULL); - self = aSelf; + delete theOptions; } void Adaptation::Icap::ServiceRep::finalize() { Adaptation::Service::finalize(); - assert(self != NULL); // use /etc/services or default port if needed const bool have_port = cfg().port >= 0; @@ -55,18 +48,6 @@ } } -void Adaptation::Icap::ServiceRep::invalidate() -{ - assert(self != NULL); - Pointer savedSelf = self; // to prevent destruction when we nullify self - self = NULL; - - announceStatusChange("invalidated by reconfigure", false); - - savedSelf = NULL; // may destroy us and, hence, invalidate cbdata(this) - // TODO: it would be nice to invalidate cbdata(this) when not destroyed -} - void Adaptation::Icap::ServiceRep::noteFailure() { ++theSessionFailures; @@ -110,7 +91,7 @@ bool Adaptation::Icap::ServiceRep::up() const { - return self != NULL && !isSuspended && hasOptions(); + return !isSuspended && hasOptions(); } bool Adaptation::Icap::ServiceRep::wantsUrl(const String &urlPath) const @@ -151,10 +132,10 @@ void Adaptation::Icap::ServiceRep::noteTimeToUpdate() { - if (self != NULL) + if (!detached()) updateScheduled = false; - if (!self || theOptionsFetcher) { + if (detached() || theOptionsFetcher) { debugs(93,5, HERE << "ignores options update " << status()); return; } @@ -200,11 +181,10 @@ debugs(93,5, HERE << "Adaptation::Icap::Service is asked to call " << *cb << " when ready " << status()); - Must(self != NULL); Must(!broken()); // we do not wait for a broken service Client i; - i.service = self; // TODO: is this really needed? + i.service = Pointer(this); // TODO: is this really needed? i.callback = cb; theClients.push_back(i); @@ -225,7 +205,7 @@ bool Adaptation::Icap::ServiceRep::needNewOptions() const { - return self != NULL && !up(); + return !detached() && !up(); } void Adaptation::Icap::ServiceRep::changeOptions(Adaptation::Icap::Options *newOptions) @@ -444,8 +424,6 @@ buf.append("up", 2); else { buf.append("down", 4); - if (!self) - buf.append(",gone", 5); if (isSuspended) buf.append(",susp", 5); @@ -457,6 +435,9 @@ buf.append(",stale", 6); } + if (detached()) + buf.append(",detached", 9); + if (theOptionsFetcher) buf.append(",fetch", 6); @@ -471,3 +452,15 @@ return buf.content(); } + +void Adaptation::Icap::ServiceRep::detach() +{ + debugs(93,3, HERE << "detaching ICAP service: " << cfg().uri << + ' ' << status()); + isDetached = true; +} + +bool Adaptation::Icap::ServiceRep::detached() const +{ + return isDetached; +} diff -u -r -N squid-3.1.3/src/adaptation/icap/ServiceRep.h squid-3.1.4/src/adaptation/icap/ServiceRep.h --- squid-3.1.3/src/adaptation/icap/ServiceRep.h 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/adaptation/icap/ServiceRep.h 2010-05-31 01:21:11.000000000 +1200 @@ -89,13 +89,10 @@ ServiceRep(const Adaptation::ServiceConfig &config); virtual ~ServiceRep(); - void setSelf(Pointer &aSelf); // needs self pointer for OptXact virtual void finalize(); - void invalidate(); // call when the service is no longer needed or valid - - bool probed() const; // see comments above - bool up() const; // see comments above + virtual bool probed() const; // see comments above + virtual bool up() const; // see comments above virtual Adaptation::Initiate *makeXactLauncher(Adaptation::Initiator *, HttpMsg *virginHeader, HttpRequest *virginCause); @@ -111,6 +108,9 @@ //AsyncJob virtual methods virtual bool doneAll() const { return Adaptation::Initiator::doneAll() && false;} + virtual void detach(); + virtual bool detached() const; + public: // treat these as private, they are for callbacks only void noteTimeToUpdate(); void noteTimeToNotify(); @@ -163,8 +163,8 @@ const char *status() const; - Pointer self; mutable bool wasAnnouncedUp; // prevent sequential same-state announcements + bool isDetached; CBDATA_CLASS2(ServiceRep); }; diff -u -r -N squid-3.1.3/src/adaptation/Initiate.cc squid-3.1.4/src/adaptation/Initiate.cc --- squid-3.1.3/src/adaptation/Initiate.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/Initiate.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section 93 ICAP (RFC 3507) Client + * DEBUG: section 93 ICAP (RFC 3507) Client */ #include "squid.h" diff -u -r -N squid-3.1.3/src/adaptation/Initiator.cc squid-3.1.4/src/adaptation/Initiator.cc --- squid-3.1.3/src/adaptation/Initiator.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/Initiator.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section 93 ICAP (RFC 3507) Client + * DEBUG: section 93 ICAP (RFC 3507) Client */ #include "squid.h" diff -u -r -N squid-3.1.3/src/adaptation/Iterator.cc squid-3.1.4/src/adaptation/Iterator.cc --- squid-3.1.3/src/adaptation/Iterator.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/Iterator.cc 2010-05-31 01:21:12.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section 93 Adaptation + * DEBUG: section 93 Adaptation */ #include "squid.h" diff -u -r -N squid-3.1.3/src/adaptation/Message.cc squid-3.1.4/src/adaptation/Message.cc --- squid-3.1.3/src/adaptation/Message.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/Message.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section XXX + * DEBUG: section 93 Adaptation */ #include "squid.h" diff -u -r -N squid-3.1.3/src/adaptation/Service.cc squid-3.1.4/src/adaptation/Service.cc --- squid-3.1.3/src/adaptation/Service.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/Service.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section XXX + * DEBUG: section 93 Adaptation */ #include "squid.h" @@ -68,3 +68,9 @@ } return NULL; } + +void Adaptation::DetachServices() +{ + while (!AllServices().empty()) + AllServices().pop_back()->detach(); +} diff -u -r -N squid-3.1.3/src/adaptation/ServiceConfig.cc squid-3.1.4/src/adaptation/ServiceConfig.cc --- squid-3.1.3/src/adaptation/ServiceConfig.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/ServiceConfig.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section XXX + * DEBUG: section 93 Adaptation */ #include "squid.h" diff -u -r -N squid-3.1.3/src/adaptation/Service.h squid-3.1.4/src/adaptation/Service.h --- squid-3.1.3/src/adaptation/Service.h 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/adaptation/Service.h 2010-05-31 01:21:11.000000000 +1200 @@ -27,9 +27,6 @@ Service(const ServiceConfig &aConfig); virtual ~Service(); - // call when the service is no longer needed or valid - virtual void invalidate() = 0; - virtual bool probed() const = 0; // see comments above virtual bool broken() const; virtual bool up() const = 0; // see comments above @@ -51,6 +48,12 @@ virtual void finalize(); // called after creation + /// called when removed from the config; the service will be + /// auto-destroyed when the last refcounting user leaves + virtual void detach() = 0; + /// whether detached() was called + virtual bool detached() const = 0; + protected: ServiceConfig &writeableCfg() { return theConfig; } @@ -64,6 +67,9 @@ extern Services &AllServices(); extern ServicePointer FindService(const Service::Id &key); +/// detach all adaptation services from current configuration +extern void DetachServices(); + } // namespace Adaptation #endif /* SQUID_ADAPTATION__SERVICE_H */ diff -u -r -N squid-3.1.3/src/base/AsyncJob.cc squid-3.1.4/src/base/AsyncJob.cc --- squid-3.1.3/src/base/AsyncJob.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/base/AsyncJob.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section 93 ICAP (RFC 3507) Client + * DEBUG: section 93 ICAP (RFC 3507) Client */ #include "squid.h" diff -u -r -N squid-3.1.3/src/cache_cf.cc squid-3.1.4/src/cache_cf.cc --- squid-3.1.3/src/cache_cf.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/cache_cf.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,7 +1,7 @@ /* * $Id$ * - * DEBUG: section 3 Configuration File Parsing + * DEBUG: section 03 Configuration File Parsing * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -2228,7 +2228,7 @@ dump_refreshpattern(StoreEntry * entry, const char *name, refresh_t * head) { while (head != NULL) { - storeAppendPrintf(entry, "%s%s %s %d %d%% %d\n", + storeAppendPrintf(entry, "%s%s %s %d %d%% %d", name, head->flags.icase ? " -i" : null_string, head->pattern, @@ -2785,7 +2785,7 @@ storeAppendPrintf(entry, "\n"); } -#include "cf_parser.h" +#include "cf_parser.cci" peer_t parseNeighborType(const char *s) diff -u -r -N squid-3.1.3/src/cf.data.pre squid-3.1.4/src/cf.data.pre --- squid-3.1.3/src/cf.data.pre 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/cf.data.pre 2010-05-31 01:21:11.000000000 +1200 @@ -740,13 +740,12 @@ If a request reaches us from a source that is allowed by this configuration item, then we consult the X-Forwarded-For header to see where that host received the request from. If the - X-Forwarded-For header contains multiple addresses, and if - acl_uses_indirect_client is on, then we continue backtracking - until we reach an address for which we are not allowed to - follow the X-Forwarded-For header, or until we reach the first - address in the list. (If acl_uses_indirect_client is off, then - it's impossible to backtrack through more than one level of - X-Forwarded-For addresses.) + X-Forwarded-For header contains multiple addresses, we continue + backtracking until we reach an address for which we are not allowed + to follow the X-Forwarded-For header, or until we reach the first + address in the list. For the purpose of ACL used in the + follow_x_forwarded_for directive the src ACL type always matches + the address we are testing and srcdomain matches its rDNS. The end result of this process is an IP address that we will refer to as the indirect client address. This address may @@ -2837,7 +2836,7 @@ NAME: client_netmask TYPE: address LOC: Config.Addrs.client_netmask -DEFAULT: 255.255.255.255 +DEFAULT: no_addr DOC_START A netmask for client addresses in logfiles and cachemgr output. Change this to protect the privacy of your cache clients. @@ -4450,7 +4449,7 @@ NAME: wccp_router TYPE: address LOC: Config.Wccp.router -DEFAULT: 0.0.0.0 +DEFAULT: any_addr IFDEF: USE_WCCP DOC_START Use this option to define your WCCP ``home'' router for @@ -4816,14 +4815,14 @@ NAME: snmp_incoming_address TYPE: address LOC: Config.Addrs.snmp_incoming -DEFAULT: 0.0.0.0 +DEFAULT: any_addr IFDEF: SQUID_SNMP DOC_NONE NAME: snmp_outgoing_address TYPE: address LOC: Config.Addrs.snmp_outgoing -DEFAULT: 255.255.255.255 +DEFAULT: no_addr IFDEF: SQUID_SNMP DOC_START Just like 'udp_incoming_address', but for the SNMP port. @@ -4833,13 +4832,13 @@ snmp_outgoing_address is used for SNMP packets returned to SNMP agents. - The default snmp_incoming_address (0.0.0.0) is to listen on all + The default snmp_incoming_address is to listen on all available network interfaces. - If snmp_outgoing_address is set to 255.255.255.255 (the default) - it will use the same socket as snmp_incoming_address. Only - change this if you want to have SNMP replies sent using another - address than where this Squid listens for SNMP queries. + If snmp_outgoing_address is not set it will use the same socket + as snmp_incoming_address. Only change this if you want to have + SNMP replies sent using another address than where this Squid + listens for SNMP queries. NOTE, snmp_incoming_address and snmp_outgoing_address can not have the same value since they both use port 3401. @@ -4891,7 +4890,7 @@ NAME: udp_incoming_address TYPE: address LOC:Config.Addrs.udp_incoming -DEFAULT: 0.0.0.0 +DEFAULT: any_addr DOC_START udp_incoming_address is used for UDP packets received from other caches. @@ -4913,7 +4912,7 @@ NAME: udp_outgoing_address TYPE: address LOC: Config.Addrs.udp_outgoing -DEFAULT: 255.255.255.255 +DEFAULT: no_addr DOC_START udp_outgoing_address is used for UDP packets sent out to other caches. @@ -5115,7 +5114,7 @@ IFDEF: MULTICAST_MISS_STREAM TYPE: address LOC: Config.mcast_miss.addr -DEFAULT: 255.255.255.255 +DEFAULT: no_addr DOC_START If you enable this option, every "cache miss" URL will be sent out on the specified multicast address. @@ -6338,7 +6337,7 @@ memory_pools_limit to a reasonably high value even if your configuration will use less memory. - If set to zero, Squid will keep all memory it can. That is, there + If set to none, Squid will keep all memory it can. That is, there will be no limit on the total amount of memory used for safe-keeping. To disable memory allocation optimization, do not set diff -u -r -N squid-3.1.3/src/cf_gen.cc squid-3.1.4/src/cf_gen.cc --- squid-3.1.3/src/cf_gen.cc 2010-05-02 22:47:09.000000000 +1200 +++ squid-3.1.4/src/cf_gen.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: none Generate squid.conf.default and cf_parser.h + * DEBUG: none Generate squid.conf.default and cf_parser.cci * AUTHOR: Max Okumoto * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -36,7 +36,7 @@ * (ie it creates the squid.conf.default file from the cf.data file) * * The output files are as follows: - * cf_parser.h - this file contains, default_all() which + * cf_parser.cci - this file contains, default_all() which * initializes variables with the default * values, parse_line() that parses line from * squid.conf.default, dump_config that dumps the @@ -48,7 +48,6 @@ //#include "squid.h" #include "config.h" //#include "SquidTime.h" -#include "cf_gen_defines.h" #include "util.h" #if HAVE_STRING_H @@ -58,8 +57,10 @@ #include #endif +#include "cf_gen_defines.cci" + #define MAX_LINE 1024 /* longest configuration line */ -#define _PATH_PARSER "cf_parser.h" +#define _PATH_PARSER "cf_parser.cci" #define _PATH_SQUID_CONF "squid.conf.documented" #define _PATH_SQUID_CONF_SHORT "squid.conf.default" #define _PATH_CF_DEPEND "cf.data.depend" diff -u -r -N squid-3.1.3/src/client_db.cc squid-3.1.4/src/client_db.cc --- squid-3.1.3/src/client_db.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/client_db.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,7 +1,7 @@ /* * $Id$ * - * DEBUG: section 0 Client Database + * DEBUG: section 00 Client Database * AUTHOR: Duane Wessels * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/comm.cc squid-3.1.4/src/comm.cc --- squid-3.1.3/src/comm.cc 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/comm.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section 5 Socket Functions + * DEBUG: section 05 Socket Functions * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -630,16 +630,6 @@ /* attempt native enabled port. */ sock = comm_openex(sock_type, proto, addr, flags, 0, note); -#if USE_IPV6 - /* under IPv6 there is the possibility IPv6 is present but disabled. */ - /* try again as IPv4-native */ - if ( sock < 0 && addr.IsIPv6() && addr.SetIPv4() ) { - /* attempt to open this IPv4-only. */ - sock = comm_openex(sock_type, proto, addr, flags, 0, note); - debugs(50, 2, HERE << "attempt open " << note << " socket on: " << addr); - } -#endif - return sock; } @@ -724,7 +714,24 @@ debugs(50, 3, "comm_openex: Attempt open socket for: " << addr ); - if ((new_socket = socket(AI->ai_family, AI->ai_socktype, AI->ai_protocol)) < 0) { + new_socket = socket(AI->ai_family, AI->ai_socktype, AI->ai_protocol); +#if USE_IPV6 + /* under IPv6 there is the possibility IPv6 is present but disabled. */ + /* try again as IPv4-native if possible */ + if ( new_socket < 0 && addr.IsIPv6() && addr.SetIPv4() ) { + /* attempt to open this IPv4-only. */ + addr.FreeAddrInfo(AI); + /* Setup the socket addrinfo details for use */ + addr.GetAddrInfo(AI); + AI->ai_socktype = sock_type; + AI->ai_protocol = proto; + debugs(50, 3, "comm_openex: Attempt fallback open socket for: " << addr ); + new_socket = socket(AI->ai_family, AI->ai_socktype, AI->ai_protocol); + debugs(50, 2, HERE << "attempt open " << note << " socket on: " << addr); + } +#endif + + if (new_socket < 0) { /* Increase the number of reserved fd's if calls to socket() * are failing because the open file table is full. This * limits the number of simultaneous clients */ @@ -968,9 +975,6 @@ int ConnectStateData::commResetFD() { - struct addrinfo *AI = NULL; - IpAddress nul; - int new_family = AF_UNSPEC; // XXX: do we have to check this? // @@ -979,14 +983,13 @@ statCounter.syscalls.sock.sockets++; - /* setup a bare-bones addrinfo */ - /* TODO INET6: for WinXP we may need to check the local_addr type and setup the family properly. */ - nul.GetAddrInfo(AI); - new_family = AI->ai_family; + fde *F = &fd_table[fd]; - int fd2 = socket(AI->ai_family, AI->ai_socktype, AI->ai_protocol); + struct addrinfo *AI = NULL; + F->local_addr.GetAddrInfo(AI); + int new_family = AI->ai_family; - nul.FreeAddrInfo(AI); + int fd2 = socket(new_family, AI->ai_socktype, AI->ai_protocol); if (fd2 < 0) { debugs(5, DBG_CRITICAL, HERE << "WARNING: FD " << fd2 << " socket failed to allocate: " << xstrerror()); @@ -994,6 +997,7 @@ if (ENFILE == errno || EMFILE == errno) fdAdjustReserved(); + F->local_addr.FreeAddrInfo(AI); return 0; } @@ -1013,17 +1017,20 @@ close(fd2); + F->local_addr.FreeAddrInfo(AI); return 0; } commResetSelect(fd); close(fd2); - fde *F = &fd_table[fd]; + + debugs(50, 3, "commResetFD: Reset socket FD " << fd << "->" << fd2 << " : family=" << new_family ); /* INET6: copy the new sockets family type to the FDE table */ - fd_table[fd].sock_family = new_family; + F->sock_family = new_family; + + F->flags.called_connect = 0; - fd_table[fd].flags.called_connect = 0; /* * yuck, this has assumptions about comm_open() arguments for * the original socket @@ -1034,9 +1041,6 @@ comm_set_transparent(fd); } - AI = NULL; - F->local_addr.GetAddrInfo(AI); - if (commBind(fd, *AI) != COMM_OK) { debugs(5, DBG_CRITICAL, "WARNING: Reset of FD " << fd << " for " << F->local_addr << " failed to bind: " << xstrerror()); F->local_addr.FreeAddrInfo(AI); @@ -1071,8 +1075,13 @@ if (squid_curtime - connstart > Config.Timeout.connect) return 0; } else { - if (tries > addrcount) + if (tries > addrcount) { + /* Flush bad address count in case we are + * skipping over incompatible protocol + */ + ipcacheMarkAllGood(host); return 0; + } } return commResetFD(); @@ -1104,8 +1113,7 @@ void ConnectStateData::connect() { - if (S.IsAnyAddr()) - defaults(); + defaults(); debugs(5,5, HERE << "to " << S); @@ -1122,15 +1130,22 @@ callCallback(COMM_OK, 0); break; -#if USE_IPV6 case COMM_ERR_PROTOCOL: + debugs(5, 5, HERE "FD " << fd << ": COMM_ERR_PROTOCOL - try again"); /* problem using the desired protocol over this socket. - * count the connection attempt, reset the socket, and immediately try again */ + * skip to the next address and hope it's more compatible + * but do not mark the current address as bad + */ tries++; - commResetFD(); - connect(); + if (commRetryConnect()) { + /* Force an addr cycle to move forward to the next possible address */ + ipcacheCycleAddr(host, NULL); + eventAdd("commReconnect", commReconnect, this, this->addrcount == 1 ? 0.05 : 0.0, 0); + } else { + debugs(5, 5, HERE << "FD " << fd << ": COMM_ERR_PROTOCOL - ERR tried too many times already."); + callCallback(COMM_ERR_CONNECT, errno); + } break; -#endif default: debugs(5, 5, HERE "FD " << fd << ": * - try again"); @@ -1232,18 +1247,27 @@ debugs(5, 9, "comm_connect_addr: connecting socket " << sock << " to " << address << " (want family: " << F->sock_family << ")"); - /* BUG 2222 FIX: reset the FD when its found to be IPv4 in IPv6 mode */ - /* inverse case of IPv4 failing to connect on IPv6 socket is handeld post-connect. + /* Handle IPv6 over IPv4-only socket case. * this case must presently be handled here since the GetAddrInfo asserts on bad mappings. - * eventually we want it to throw a Must() that gets handled there instead of this if. - * NP: because commresetFD is private to ConnStateData we have to return an error and + * NP: because commResetFD is private to ConnStateData we have to return an error and * trust its handled properly. */ -#if USE_IPV6 if (F->sock_family == AF_INET && !address.IsIPv4()) { + errno = ENETUNREACH; + return COMM_ERR_PROTOCOL; + } + + /* Handle IPv4 over IPv6-only socket case. + * This case is presently handled here as it's both a known case and it's + * uncertain what error will be returned by the IPv6 stack in such case. It's + * possible this will also be handled by the errno checks below after connect() + * but needs carefull cross-platform verification, and verifying the address + * condition here is simple. + */ + if (!F->local_addr.IsIPv4() && address.IsIPv4()) { + errno = ENETUNREACH; return COMM_ERR_PROTOCOL; } -#endif address.GetAddrInfo(AI, F->sock_family); @@ -1337,23 +1361,10 @@ status = COMM_OK; else if (ignoreErrno(errno)) status = COMM_INPROGRESS; + else if (errno == EAFNOSUPPORT || errno == EINVAL) + return COMM_ERR_PROTOCOL; else -#if USE_IPV6 - if ( address.IsIPv4() && F->sock_family == AF_INET6 ) { - - /* failover to trying IPv4-only link if an IPv6 one fails */ - /* to catch the edge case of apps listening on IPv4-localhost */ - F->sock_family = AF_INET; - int res = comm_connect_addr(sock, address); - - /* if that fails too, undo our temporary socktype hack so the repeat works properly. */ - if (res == COMM_ERROR) - F->sock_family = AF_INET6; - - return res; - } else -#endif - return COMM_ERROR; + return COMM_ERROR; address.NtoA(F->ipaddr, MAX_IPSTRLEN); @@ -1424,9 +1435,9 @@ F->remote_port = details.peer.GetPort(); F->local_addr.SetPort(details.me.GetPort()); #if USE_IPV6 - F->sock_family = AF_INET; -#else F->sock_family = details.me.IsIPv4()?AF_INET:AF_INET6; +#else + F->sock_family = AF_INET; #endif details.me.FreeAddrInfo(gai); diff -u -r -N squid-3.1.3/src/comm_epoll.cc squid-3.1.4/src/comm_epoll.cc --- squid-3.1.3/src/comm_epoll.cc 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/comm_epoll.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,8 +1,7 @@ - /* * $Id$ * - * DEBUG: section 5 Socket Functions + * DEBUG: section 05 Socket Functions * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- diff -u -r -N squid-3.1.3/src/comm.h squid-3.1.4/src/comm.h --- squid-3.1.3/src/comm.h 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/comm.h 2010-05-31 01:21:12.000000000 +1200 @@ -22,9 +22,7 @@ COMM_ERR_CONNECT = -8, COMM_ERR_DNS = -9, COMM_ERR_CLOSING = -10, -#if USE_IPV6 COMM_ERR_PROTOCOL = -11, /* IPv4 or IPv6 cannot be used on the fd socket */ -#endif COMM_ERR__END__ = -999999 /* Dummy entry to make syntax valid (comma on line above), do not use. New entries added above */ } comm_err_t; diff -u -r -N squid-3.1.3/src/comm_kqueue.cc squid-3.1.4/src/comm_kqueue.cc --- squid-3.1.3/src/comm_kqueue.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/comm_kqueue.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,8 +1,7 @@ - /* * $Id$ * - * DEBUG: section 5 Socket Functions + * DEBUG: section 05 Socket Functions * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- diff -u -r -N squid-3.1.3/src/comm_poll.cc squid-3.1.4/src/comm_poll.cc --- squid-3.1.3/src/comm_poll.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/comm_poll.cc 2010-05-31 01:21:12.000000000 +1200 @@ -2,7 +2,7 @@ /* * $Id$ * - * DEBUG: section 5 Socket Functions + * DEBUG: section 05 Socket Functions * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- diff -u -r -N squid-3.1.3/src/CommRead.h squid-3.1.4/src/CommRead.h --- squid-3.1.3/src/CommRead.h 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/CommRead.h 2010-05-31 01:21:11.000000000 +1200 @@ -1,8 +1,7 @@ - /* * $Id$ * - * DEBUG: section 5 Comms + * DEBUG: section 05 Comm * AUTHOR: Robert Collins * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/comm_select.cc squid-3.1.4/src/comm_select.cc --- squid-3.1.3/src/comm_select.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/comm_select.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,8 +1,7 @@ - /* * $Id$ * - * DEBUG: section 5 Socket Functions + * DEBUG: section 05 Socket Functions * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- diff -u -r -N squid-3.1.3/src/comm_select_win32.cc squid-3.1.4/src/comm_select_win32.cc --- squid-3.1.3/src/comm_select_win32.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/comm_select_win32.cc 2010-05-31 01:21:12.000000000 +1200 @@ -1,8 +1,7 @@ - /* * $Id$ * - * DEBUG: section 5 Socket Functions + * DEBUG: section 05 Socket Functions * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- diff -u -r -N squid-3.1.3/src/ConfigOption.cc squid-3.1.4/src/ConfigOption.cc --- squid-3.1.3/src/ConfigOption.cc 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/ConfigOption.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,8 +1,7 @@ - /* * $Id$ * - * DEBUG: section 3 Configuration File Parsing + * DEBUG: section 03 Configuration File Parsing * AUTHOR: Robert Collins * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/ConnectionDetail.h squid-3.1.4/src/ConnectionDetail.h --- squid-3.1.3/src/ConnectionDetail.h 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/ConnectionDetail.h 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section 5 Socket Functions + * DEBUG: section 05 Socket Functions * AUTHOR: Robert Collins * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/debug.cc squid-3.1.4/src/debug.cc --- squid-3.1.3/src/debug.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/debug.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,7 +1,7 @@ /* * $Id$ * - * DEBUG: section 0 Debug Routines + * DEBUG: section 00 Debug Routines * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/Debug.h squid-3.1.4/src/Debug.h --- squid-3.1.3/src/Debug.h 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/Debug.h 2010-05-31 01:21:11.000000000 +1200 @@ -1,7 +1,7 @@ /* * $Id$ * - * DEBUG: section 0 Debug Routines + * DEBUG: section 00 Debug Routines * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/DelayConfig.h squid-3.1.4/src/DelayConfig.h --- squid-3.1.3/src/DelayConfig.h 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/DelayConfig.h 2010-05-31 01:21:11.000000000 +1200 @@ -1,7 +1,7 @@ /* * $Id$ * - * DEBUG: section 3 Configuration Settings + * DEBUG: section 03 Configuration Settings * AUTHOR: Robert Collins * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/DescriptorSet.cc squid-3.1.4/src/DescriptorSet.cc --- squid-3.1.3/src/DescriptorSet.cc 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/DescriptorSet.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,8 +1,7 @@ - /* * $Id$ * - * DEBUG: section 5 Comm + * DEBUG: section 05 Comm */ #include "DescriptorSet.h" diff -u -r -N squid-3.1.3/src/disk.cc squid-3.1.4/src/disk.cc --- squid-3.1.3/src/disk.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/disk.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,8 +1,7 @@ - /* * $Id$ * - * DEBUG: section 6 Disk I/O Routines + * DEBUG: section 06 Disk I/O Routines * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/dnsserver.cc squid-3.1.4/src/dnsserver.cc --- squid-3.1.3/src/dnsserver.cc 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/dnsserver.cc 2010-05-31 01:21:12.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section 0 DNS Resolver + * DEBUG: section 00 DNS Resolver Daemon * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/errorpage.cc squid-3.1.4/src/errorpage.cc --- squid-3.1.3/src/errorpage.cc 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/errorpage.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,8 +1,7 @@ - /* * $Id$ * - * DEBUG: section 4 Error Generation + * DEBUG: section 04 Error Generation * AUTHOR: Duane Wessels * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/errorpage.h squid-3.1.4/src/errorpage.h --- squid-3.1.3/src/errorpage.h 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/errorpage.h 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section 4 Error Generation + * DEBUG: section 04 Error Generation * AUTHOR: Duane Wessels * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/EventLoop.cc squid-3.1.4/src/EventLoop.cc --- squid-3.1.3/src/EventLoop.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/EventLoop.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,8 +1,7 @@ - /* * $Id$ * - * DEBUG: section 1 Main Loop + * DEBUG: section 01 Main Loop * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/filemap.cc squid-3.1.4/src/filemap.cc --- squid-3.1.3/src/filemap.cc 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/filemap.cc 2010-05-31 01:21:12.000000000 +1200 @@ -1,8 +1,7 @@ - /* * $Id$ * - * DEBUG: section 8 Swap File Bitmap + * DEBUG: section 08 Swap File Bitmap * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/forward.cc squid-3.1.4/src/forward.cc --- squid-3.1.3/src/forward.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/forward.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1360,7 +1360,12 @@ ch.dst_peer = dst_peer; if (request) { - ch.src_addr = request->client_addr; +#if FOLLOW_X_FORWARDED_FOR + if (Config.onoff.acl_uses_indirect_client) + ch.src_addr = request->indirect_client_addr; + else +#endif + ch.src_addr = request->client_addr; ch.my_addr = request->my_addr; } diff -u -r -N squid-3.1.3/src/ftp.cc squid-3.1.4/src/ftp.cc --- squid-3.1.3/src/ftp.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/ftp.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,7 +1,7 @@ /* * $Id$ * - * DEBUG: section 9 File Transfer Protocol (FTP) + * DEBUG: section 09 File Transfer Protocol (FTP) * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1670,6 +1670,12 @@ ctrl.last_command = ebuf; + if (!canSend(ctrl.fd)) { + debugs(9, 2, HERE << "cannot send to closing ctrl FD " << ctrl.fd); + // TODO: assert(ctrl.closer != NULL); + return; + } + typedef CommCbMemFunT Dialer; AsyncCall::Pointer call = asyncCall(9, 5, "FtpStateData::ftpWriteCommandCallback", Dialer(this, &FtpStateData::ftpWriteCommandCallback)); diff -u -r -N squid-3.1.3/src/http.cc squid-3.1.4/src/http.cc --- squid-3.1.3/src/http.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/http.cc 2010-05-31 01:21:11.000000000 +1200 @@ -675,6 +675,9 @@ assert(!flags.headers_parsed); + if (!readBuf->hasContent()) + return; + http_status error = HTTP_STATUS_NONE; HttpReply *newrep = new HttpReply; @@ -1126,12 +1129,11 @@ eof = 1; flags.do_next_read = 0; - /* Bug 2789: Replies may terminate with \r\n then EOF instead of \r\n\r\n + /* Bug 2879: Replies may terminate with \r\n then EOF instead of \r\n\r\n * Ensure here that we have at minimum two \r\n when EOF is seen. - * TODO: When headersEnd() is cleaned up to only be called once we can merge - * this as a special case there where it belongs. + * TODO: Add eof parameter to headersEnd() and move this hack there. */ - if (!flags.headers_parsed) { + if (readBuf->contentSize() && !flags.headers_parsed) { /* * Yes Henrik, there is a point to doing this. When we * called httpProcessReplyHeader() before, we didn't find @@ -1140,7 +1142,6 @@ */ /* Fake an "end-of-headers" to work around such broken servers */ readBuf->append("\r\n", 2); - len = 2; } } @@ -1207,7 +1208,8 @@ debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: Headers did not parse at all for " << entry->url() << " AKA " << orig_request->GetHost() << orig_request->urlpath.termedBuf() ); } else { error = ERR_ZERO_SIZE_OBJECT; - debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: No object data received for " << entry->url() << " AKA " << orig_request->GetHost() << orig_request->urlpath.termedBuf() ); + debugs(11, (orig_request->flags.accelerated?DBG_IMPORTANT:2), "WARNING: HTTP: Invalid Response: No object data received for " << + entry->url() << " AKA " << orig_request->GetHost() << orig_request->urlpath.termedBuf() ); } } @@ -1979,6 +1981,13 @@ MemBuf mb; debugs(11, 5, "httpSendRequest: FD " << fd << ", request " << request << ", this " << this << "."); + + if (!canSend(fd)) { + debugs(11,3, HERE << "cannot send request to closing FD " << fd); + assert(closeHandler != NULL); + return false; + } + typedef CommCbMemFunT TimeoutDialer; AsyncCall::Pointer timeoutCall = asyncCall(11, 5, "HttpStateData::httpTimeout", TimeoutDialer(this,&HttpStateData::httpTimeout)); @@ -2081,6 +2090,13 @@ sendComplete(io); } else { debugs(11, 2, "doneSendingRequestBody: matched brokenPosts"); + + if (!canSend(fd)) { + debugs(11,2, HERE << "cannot send CRLF to closing FD " << fd); + assert(closeHandler != NULL); + return; + } + typedef CommCbMemFunT Dialer; Dialer dialer(this, &HttpStateData::sendComplete); AsyncCall::Pointer call= asyncCall(11,5, "HttpStateData::SendComplete", dialer); diff -u -r -N squid-3.1.3/src/HttpHeader.cc squid-3.1.4/src/HttpHeader.cc --- squid-3.1.3/src/HttpHeader.cc 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/HttpHeader.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1534,8 +1534,6 @@ Headers[id].stat.seenCount++; - Headers[id].stat.aliveCount++; - debugs(55, 9, "parsed HttpHeaderEntry: '" << name << ": " << value << "'"); return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf()); diff -u -r -N squid-3.1.3/src/icmp/IcmpConfig.h squid-3.1.4/src/icmp/IcmpConfig.h --- squid-3.1.3/src/icmp/IcmpConfig.h 2010-05-02 22:47:09.000000000 +1200 +++ squid-3.1.4/src/icmp/IcmpConfig.h 2010-05-31 01:21:11.000000000 +1200 @@ -1,7 +1,7 @@ /* * $Id$ * - * DEBUG: section 3 Configuration Settings + * DEBUG: section 03 Configuration Settings * AUTHOR: Amos Jeffries * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/ip/IpAddress.cc squid-3.1.4/src/ip/IpAddress.cc --- squid-3.1.3/src/ip/IpAddress.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/ip/IpAddress.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section 14 IP Storage and Handling + * DEBUG: section 14 IP Storage and Handling * AUTHOR: Amos Jeffries * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -168,9 +168,6 @@ p1[i] &= p2[i]; } - /* we have found a situation where mask forms or destroys a IPv4 map. */ - check4Mapped(); - return changes; } @@ -288,6 +285,9 @@ const struct in6_addr IpAddress::v4_anyaddr = {{{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }} }; +const struct in6_addr IpAddress::v4_noaddr = {{{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }} +}; const struct in6_addr IpAddress::v6_noaddr = {{{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }} }; @@ -511,9 +511,6 @@ memcpy(&m_SocketAddr, &s, sizeof(struct sockaddr_in)); #endif - /* maintain stored family values properly */ - check4Mapped(); - return *this; }; @@ -534,12 +531,6 @@ return *this; }; -void IpAddress::check4Mapped() -{ - // obsolete. - // TODO use this NOW to set the sin6_family properly on exporting. not on import. -} - #if USE_IPV6 IpAddress::IpAddress(struct sockaddr_in6 const &s) { @@ -551,8 +542,6 @@ { memcpy(&m_SocketAddr, &s, sizeof(struct sockaddr_in6)); - /* maintain address family properly */ - check4Mapped(); return *this; }; @@ -575,10 +564,6 @@ memcpy(&m_SocketAddr.sin_addr, &s, sizeof(struct in_addr)); #endif - - /* maintain stored family type properly */ - check4Mapped(); - return *this; }; @@ -596,9 +581,6 @@ memcpy(&m_SocketAddr.sin6_addr, &s, sizeof(struct in6_addr)); m_SocketAddr.sin6_family = AF_INET6; - /* maintain address family type properly */ - check4Mapped(); - return *this; }; @@ -996,11 +978,11 @@ p++; } - /* 7 being space for [,], and port */ + /* 8 being space for [ ] : and port digits */ if ( IsIPv6() ) - NtoA(p, blen-7, AF_INET6); + NtoA(p, blen-8, AF_INET6); else - NtoA(p, blen-7, AF_INET); + NtoA(p, blen-8, AF_INET); // find the end of the new string while (*p != '\0' && p < buf+blen) @@ -1030,9 +1012,9 @@ p += ToHostname(p, blen); - if (m_SocketAddr.sin6_port > 0 && p < (buf+blen-6) ) { - /* 6 is max length of expected ':port' (short int) */ - snprintf(p, 6,":%d", GetPort() ); + if (m_SocketAddr.sin6_port > 0 && p <= (buf+blen-7) ) { + // ':port' (short int) needs at most 6 bytes plus 1 for 0-terminator + snprintf(p, 7, ":%d", GetPort() ); } // force a null-terminated string @@ -1123,15 +1105,13 @@ if ( in.s_addr == 0x00000000) { /* ANYADDR */ - memset(&out, 0, sizeof(struct in6_addr)); + out = v4_anyaddr; } else if ( in.s_addr == 0xFFFFFFFF) { /* NOADDR */ - memset(&out, 255, sizeof(struct in6_addr)); + out = v4_noaddr; } else { /* general */ - memset(&out, 0, sizeof(struct in6_addr)); - out.s6_addr[10] = 0xFF; - out.s6_addr[11] = 0xFF; + out = v4_anyaddr; out.s6_addr[12] = ((uint8_t *)&in.s_addr)[0]; out.s6_addr[13] = ((uint8_t *)&in.s_addr)[1]; out.s6_addr[14] = ((uint8_t *)&in.s_addr)[2]; diff -u -r -N squid-3.1.3/src/ip/IpAddress.h squid-3.1.4/src/ip/IpAddress.h --- squid-3.1.3/src/ip/IpAddress.h 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/ip/IpAddress.h 2010-05-31 01:21:11.000000000 +1200 @@ -390,8 +390,6 @@ bool GetReverseString4(char buf[MAX_IPSTRLEN], const struct in_addr &dat) const; - void check4Mapped(); - #if USE_IPV6 bool GetReverseString6(char buf[MAX_IPSTRLEN], const struct in6_addr &dat) const; @@ -426,6 +424,7 @@ static const unsigned int MAX_IP6_STRLEN = STRLEN_IP6R; static const struct in6_addr v4_localhost; static const struct in6_addr v4_anyaddr; + static const struct in6_addr v4_noaddr; static const struct in6_addr v6_noaddr; #endif }; diff -u -r -N squid-3.1.3/src/ipcache.cc squid-3.1.4/src/ipcache.cc --- squid-3.1.3/src/ipcache.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/ipcache.cc 2010-05-31 01:21:12.000000000 +1200 @@ -1354,6 +1354,28 @@ /// \ingroup IPCacheAPI void +ipcacheMarkAllGood(const char *name) +{ + ipcache_entry *i; + ipcache_addrs *ia; + int k; + + if ((i = ipcache_get(name)) == NULL) + return; + + ia = &i->addrs; + + /* All bad, reset to All good */ + debugs(14, 3, "ipcacheMarkAllGood: Changing ALL " << name << " addrs to OK (" << ia->badcount << "/" << ia->count << " bad)"); + + for (k = 0; k < ia->count; k++) + ia->bad_mask[k] = 0; + + ia->badcount = 0; +} + +/// \ingroup IPCacheAPI +void ipcacheMarkGoodAddr(const char *name, IpAddress &addr) { ipcache_entry *i; diff -u -r -N squid-3.1.3/src/main.cc squid-3.1.4/src/main.cc --- squid-3.1.3/src/main.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/main.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,7 +1,7 @@ /* * $Id$ * - * DEBUG: section 1 Startup and Main Loop + * DEBUG: section 01 Startup and Main Loop * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -736,6 +736,24 @@ parseEtcHosts(); errorInitialize(); /* reload error pages */ accessLogInit(); + +#if USE_LOADABLE_MODULES + LoadableModulesConfigure(Config.loadable_module_names); +#endif + +#if USE_ADAPTATION + bool enableAdaptation = false; +#if ICAP_CLIENT + Adaptation::Icap::TheConfig.finalize(); + enableAdaptation = Adaptation::Icap::TheConfig.onoff || enableAdaptation; +#endif +#if USE_ECAP + Adaptation::Ecap::TheConfig.finalize(); // must be after we load modules + enableAdaptation = Adaptation::Ecap::TheConfig.onoff || enableAdaptation; +#endif + Adaptation::Config::Finalize(enableAdaptation); +#endif + #if ICAP_CLIENT icapLogOpen(); #endif diff -u -r -N squid-3.1.3/src/Makefile.am squid-3.1.4/src/Makefile.am --- squid-3.1.3/src/Makefile.am 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/Makefile.am 2010-05-31 01:21:11.000000000 +1200 @@ -193,16 +193,16 @@ $(UNLINKD) cf_gen_SOURCES = cf_gen.cc -nodist_cf_gen_HEADER = cf_gen_defines.h +nodist_cf_gen_HEADER = cf_gen_defines.cci cf_gen_LDADD= \ ../compat/libcompat.la \ -L../lib -lmiscutil \ $(EPOLL_LIBS) \ $(MINGW_LIBS) \ $(XTRA_LIBS) -cf_gen.$(OBJEXT): cf_gen_defines.h +cf_gen.$(OBJEXT): cf_gen_defines.cci -## cf_gen.cc needs src/cf_gen_defines.h +## cf_gen.cc needs src/cf_gen_defines.cci INCLUDES += -I$(top_builddir)/src @@ -510,7 +510,7 @@ nodist_squid_SOURCES = \ $(DISKIO_GEN_SOURCE) \ repl_modules.cc \ - cf_parser.h \ + cf_parser.cci \ globals.cc \ string_arrays.c @@ -630,8 +630,8 @@ globals.cc BUILT_SOURCES = \ - cf_gen_defines.h \ - cf_parser.h \ + cf_gen_defines.cci \ + cf_parser.cci \ globals.cc \ string_arrays.c \ repl_modules.cc @@ -745,16 +745,16 @@ $(CC) -o $@ $(LDFLAGS) $@.o CacheDigest.o debug.o globals.o store_key_md5.o $(STD_APP_LIBS) ## If autodependency works well this is not needed anymore -cache_cf.o: cf_parser.h +cache_cf.o: cf_parser.cci -# squid.conf.default is built by cf_gen when making cf_parser.h -squid.conf.default squid.conf.documented: cf_parser.h +# squid.conf.default is built by cf_gen when making cf_parser.cci +squid.conf.default squid.conf.documented: cf_parser.cci true -cf_parser.h: cf.data cf_gen$(EXEEXT) +cf_parser.cci: cf.data cf_gen$(EXEEXT) ./cf_gen cf.data $(srcdir)/cf.data.depend -cf_gen_defines.h: $(srcdir)/cf_gen_defines $(srcdir)/cf.data.pre +cf_gen_defines.cci: $(srcdir)/cf_gen_defines $(srcdir)/cf.data.pre $(AWK) -f $(srcdir)/cf_gen_defines <$(srcdir)/cf.data.pre >$@ || ($(RM) -f $@ && exit 1) @@ -811,7 +811,7 @@ @$(SHELL) $(top_srcdir)/scripts/remove-cfg.sh "$(RM)" $(DESTDIR)$(DEFAULT_MIME_TABLE) $(srcdir)/mime.conf.default @$(SHELL) $(top_srcdir)/scripts/remove-cfg.sh "$(RM)" $(DESTDIR)$(DEFAULT_CONFIG_FILE) squid.conf.default -CLEANFILES += cf_gen_defines.h cf.data cf_parser.h squid.conf.default squid.conf.documented \ +CLEANFILES += cf_gen_defines.cci cf.data cf_parser.cci squid.conf.default squid.conf.documented \ globals.cc string_arrays.c repl_modules.cc DiskIO/DiskIOModules_gen.cc \ test_tools.cc *.a diff -u -r -N squid-3.1.3/src/Makefile.in squid-3.1.4/src/Makefile.in --- squid-3.1.3/src/Makefile.in 2010-05-02 22:47:35.000000000 +1200 +++ squid-3.1.4/src/Makefile.in 2010-05-31 01:21:32.000000000 +1200 @@ -1530,10 +1530,10 @@ top_srcdir = @top_srcdir@ AM_CFLAGS = $(SQUID_CFLAGS) AM_CXXFLAGS = $(SQUID_CXXFLAGS) -CLEANFILES = cf_gen_defines.h cf.data cf_parser.h squid.conf.default \ - squid.conf.documented globals.cc string_arrays.c \ - repl_modules.cc DiskIO/DiskIOModules_gen.cc test_tools.cc *.a \ - testHeaders +CLEANFILES = cf_gen_defines.cci cf.data cf_parser.cci \ + squid.conf.default squid.conf.documented globals.cc \ + string_arrays.c repl_modules.cc DiskIO/DiskIOModules_gen.cc \ + test_tools.cc *.a testHeaders TESTS = $(check_PROGRAMS) testHeaders INCLUDES = -I$(top_srcdir) -I$(top_srcdir)/include -I$(top_srcdir)/src \ -I$(top_builddir)/include $(SQUID_CPPUNIT_INC) \ @@ -1646,7 +1646,7 @@ fs/libfs.la cf_gen_SOURCES = cf_gen.cc -nodist_cf_gen_HEADER = cf_gen_defines.h +nodist_cf_gen_HEADER = cf_gen_defines.cci cf_gen_LDADD = \ ../compat/libcompat.la \ -L../lib -lmiscutil \ @@ -1788,7 +1788,7 @@ nodist_squid_SOURCES = \ $(DISKIO_GEN_SOURCE) \ repl_modules.cc \ - cf_parser.h \ + cf_parser.cci \ globals.cc \ string_arrays.c @@ -1869,8 +1869,8 @@ globals.cc BUILT_SOURCES = \ - cf_gen_defines.h \ - cf_parser.h \ + cf_gen_defines.cci \ + cf_parser.cci \ globals.cc \ string_arrays.c \ repl_modules.cc @@ -4514,7 +4514,7 @@ $(OBJS): $(top_srcdir)/include/version.h $(top_builddir)/include/autoconf.h -cf_gen.$(OBJEXT): cf_gen_defines.h +cf_gen.$(OBJEXT): cf_gen_defines.cci DiskIO/DiskIOModules_gen.cc: Makefile $(SHELL) $(srcdir)/DiskIO/modules.sh $(DISK_MODULES) > DiskIO/DiskIOModules_gen.cc @@ -4533,16 +4533,16 @@ test_cache_digest: test_cache_digest.o CacheDigest.o debug.o globals.o store_key_md5.o $(CC) -o $@ $(LDFLAGS) $@.o CacheDigest.o debug.o globals.o store_key_md5.o $(STD_APP_LIBS) -cache_cf.o: cf_parser.h +cache_cf.o: cf_parser.cci -# squid.conf.default is built by cf_gen when making cf_parser.h -squid.conf.default squid.conf.documented: cf_parser.h +# squid.conf.default is built by cf_gen when making cf_parser.cci +squid.conf.default squid.conf.documented: cf_parser.cci true -cf_parser.h: cf.data cf_gen$(EXEEXT) +cf_parser.cci: cf.data cf_gen$(EXEEXT) ./cf_gen cf.data $(srcdir)/cf.data.depend -cf_gen_defines.h: $(srcdir)/cf_gen_defines $(srcdir)/cf.data.pre +cf_gen_defines.cci: $(srcdir)/cf_gen_defines $(srcdir)/cf.data.pre $(AWK) -f $(srcdir)/cf_gen_defines <$(srcdir)/cf.data.pre >$@ || ($(RM) -f $@ && exit 1) cf.data: cf.data.pre Makefile diff -u -r -N squid-3.1.3/src/mem.cc squid-3.1.4/src/mem.cc --- squid-3.1.3/src/mem.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/mem.cc 2010-05-31 01:21:11.000000000 +1200 @@ -353,8 +353,11 @@ new_pool_limit = 0; else if (Config.MemPools.limit > 0) new_pool_limit = Config.MemPools.limit; - else - new_pool_limit = mem_unlimited_size; + else { + if (Config.MemPools.limit == 0) + debugs(13, 1, "memory_pools_limit 0 has been chagned to memory_pools_limit none. Please update your config"); + new_pool_limit = -1; + } #if 0 /** \par @@ -580,27 +583,38 @@ MemPoolMeter *pm = mp_st->meter; const char *delim = "\t "; +#if HAVE_IOMANIP + stream.setf(std::ios_base::fixed); +#endif stream << std::setw(20) << std::left << mp_st->label << delim; stream << std::setw(4) << std::right << mp_st->obj_size << delim; /* Chunks */ - stream << std::setw(4) << toKB(mp_st->obj_size * mp_st->chunk_capacity) << delim; - stream << std::setw(4) << mp_st->chunk_capacity << delim; - if (mp_st->chunk_capacity) { + stream << std::setw(4) << toKB(mp_st->obj_size * mp_st->chunk_capacity) << delim; + stream << std::setw(4) << mp_st->chunk_capacity << delim; + needed = mp_st->items_inuse / mp_st->chunk_capacity; if (mp_st->items_inuse % mp_st->chunk_capacity) needed++; excess = mp_st->chunks_inuse - needed; - } - stream << std::setw(4) << mp_st->chunks_alloc << delim; - stream << std::setw(4) << mp_st->chunks_inuse << delim; - stream << std::setw(4) << mp_st->chunks_free << delim; - stream << std::setw(4) << mp_st->chunks_partial << delim; - stream << std::setprecision(3) << xpercent(excess, needed) << delim; + stream << std::setw(4) << mp_st->chunks_alloc << delim; + stream << std::setw(4) << mp_st->chunks_inuse << delim; + stream << std::setw(4) << mp_st->chunks_free << delim; + stream << std::setw(4) << mp_st->chunks_partial << delim; + stream << std::setprecision(3) << xpercent(excess, needed) << delim; + } else { + stream << delim; + stream << delim; + stream << delim; + stream << delim; + stream << delim; + stream << delim; + stream << delim; + } /* * Fragmentation calculation: * needed = inuse.level / chunk_capacity @@ -627,10 +641,10 @@ stream << toKB(mp_st->obj_size * pm->idle.hwater_level) << delim; /* saved */ stream << (int)pm->gb_saved.count << delim; - stream << std::setprecision(3) << xpercent(pm->gb_saved.count, AllMeter->gb_saved.count) << delim; - stream << std::setprecision(3) << xpercent(pm->gb_saved.bytes, AllMeter->gb_saved.bytes) << delim; - stream << std::setprecision(3) << xdiv(pm->gb_saved.count - pm->gb_osaved.count, xm_deltat) << "\n"; - pm->gb_osaved.count = pm->gb_saved.count; + stream << std::setprecision(3) << xpercent(pm->gb_saved.count, AllMeter->gb_allocated.count) << delim; + stream << std::setprecision(3) << xpercent(pm->gb_saved.bytes, AllMeter->gb_allocated.bytes) << delim; + stream << std::setprecision(3) << xdiv(pm->gb_allocated.count - pm->gb_oallocated.count, xm_deltat) << "\n"; + pm->gb_oallocated.count = pm->gb_allocated.count; } static int @@ -683,7 +697,7 @@ "In Use\t\t\t\t\t" "Idle\t\t\t" "Allocations Saved\t\t\t" - "Hit Rate\t" + "Rate\t" "\n" " \t (bytes)\t" "KB/ch\t obj/ch\t" @@ -692,7 +706,7 @@ "(#)\t (KB)\t high (KB)\t high (hrs)\t %alloc\t" "(#)\t (KB)\t high (KB)\t" "(#)\t %cnt\t %vol\t" - "(#) / sec\t" + "(#)/sec\t" "\n"; xm_deltat = current_dtime - xm_time; xm_time = current_dtime; @@ -712,7 +726,7 @@ if (!mp_stats.pool) /* pool destroyed */ continue; - if (mp_stats.pool->getMeter().gb_saved.count > 0) /* this pool has been used */ + if (mp_stats.pool->getMeter().gb_allocated.count > 0) /* this pool has been used */ sortme[npools++] = mp_stats; else not_used++; @@ -746,12 +760,13 @@ PoolReport(&mp_stats, mp_total.TheMeter, stream); /* Cumulative */ - stream << "Cumulative allocated volume: "<< double_to_str(buf, 64, mp_total.TheMeter->gb_saved.bytes) << "\n"; + stream << "Cumulative allocated volume: "<< double_to_str(buf, 64, mp_total.TheMeter->gb_allocated.bytes) << "\n"; /* overhead */ stream << "Current overhead: " << mp_total.tot_overhead << " bytes (" << std::setprecision(3) << xpercent(mp_total.tot_overhead, mp_total.TheMeter->inuse.level) << "%)\n"; /* limits */ - stream << "Idle pool limit: " << std::setprecision(2) << toMB(mp_total.mem_idle_limit) << " MB\n"; + if (mp_total.mem_idle_limit >= 0) + stream << "Idle pool limit: " << std::setprecision(2) << toMB(mp_total.mem_idle_limit) << " MB\n"; /* limits */ stream << "Total Pools created: " << mp_total.tot_pools_alloc << "\n"; stream << "Pools ever used: " << mp_total.tot_pools_alloc - not_used << " (shown above)\n"; diff -u -r -N squid-3.1.3/src/multicast.cc squid-3.1.4/src/multicast.cc --- squid-3.1.3/src/multicast.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/multicast.cc 2010-05-31 01:21:11.000000000 +1200 @@ -2,7 +2,7 @@ /* * $Id$ * - * DEBUG: section 7 Multicast + * DEBUG: section 07 Multicast * AUTHOR: Martin Hamilton * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/Parsing.cc squid-3.1.4/src/Parsing.cc --- squid-3.1.3/src/Parsing.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/Parsing.cc 2010-05-31 01:21:11.000000000 +1200 @@ -2,7 +2,7 @@ /* * $Id$ * - * DEBUG: section 3 Configuration File Parsing + * DEBUG: section 03 Configuration File Parsing * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/Parsing.h squid-3.1.4/src/Parsing.h --- squid-3.1.3/src/Parsing.h 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/Parsing.h 2010-05-31 01:21:11.000000000 +1200 @@ -2,7 +2,7 @@ /* * $Id$ * - * DEBUG: section 3 Configuration File Parsing + * DEBUG: section 03 Configuration File Parsing * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/protos.h squid-3.1.4/src/protos.h --- squid-3.1.3/src/protos.h 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/protos.h 2010-05-31 01:21:11.000000000 +1200 @@ -330,10 +330,9 @@ SQUIDCEXTERN void ipcache_init(void); SQUIDCEXTERN void stat_ipcache_get(StoreEntry *); SQUIDCEXTERN void ipcacheCycleAddr(const char *name, ipcache_addrs *); - SQUIDCEXTERN void ipcacheMarkBadAddr(const char *name, IpAddress &); - SQUIDCEXTERN void ipcacheMarkGoodAddr(const char *name, IpAddress &); +SQUIDCEXTERN void ipcacheMarkAllGood(const char *name); SQUIDCEXTERN void ipcacheFreeMemory(void); SQUIDCEXTERN ipcache_addrs *ipcacheCheckNumeric(const char *name); SQUIDCEXTERN void ipcache_restart(void); diff -u -r -N squid-3.1.3/src/recv-announce.cc squid-3.1.4/src/recv-announce.cc --- squid-3.1.3/src/recv-announce.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/recv-announce.cc 2010-05-31 01:21:11.000000000 +1200 @@ -2,7 +2,7 @@ /* * $Id$ * - * DEBUG: section 0 Announcement Server + * DEBUG: section 00 Announcement Server * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/Server.cc squid-3.1.4/src/Server.cc --- squid-3.1.3/src/Server.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/Server.cc 2010-05-31 01:21:11.000000000 +1200 @@ -35,6 +35,7 @@ #include "squid.h" #include "Server.h" #include "Store.h" +#include "fde.h" /* for fd_table[fd].closing */ #include "HttpRequest.h" #include "HttpReply.h" #include "TextException.h" @@ -393,18 +394,32 @@ sendMoreRequestBody(); } +bool +ServerStateData::canSend(int fd) const +{ + return fd >= 0 && !fd_table[fd].closing(); +} + void ServerStateData::sendMoreRequestBody() { assert(requestBodySource != NULL); assert(!requestSender); + + const int fd = dataDescriptor(); + + if (!canSend(fd)) { + debugs(9,3, HERE << "cannot send request body to closing FD " << fd); + return; // wait for the kid's close handler; TODO: assert(closer); + } + MemBuf buf; if (requestBodySource->getMoreData(buf)) { debugs(9,3, HERE << "will write " << buf.contentSize() << " request body bytes"); typedef CommCbMemFunT Dialer; requestSender = asyncCall(93,3, "ServerStateData::sentRequestBody", Dialer(this, &ServerStateData::sentRequestBody)); - comm_write_mbuf(dataDescriptor(), &buf, requestSender); + comm_write_mbuf(fd, &buf, requestSender); } else { debugs(9,3, HERE << "will wait for more request body bytes or eof"); requestSender = NULL; diff -u -r -N squid-3.1.3/src/Server.h squid-3.1.4/src/Server.h --- squid-3.1.3/src/Server.h 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/Server.h 2010-05-31 01:21:11.000000000 +1200 @@ -128,6 +128,8 @@ void handleRequestBodyProductionEnded(); virtual void handleRequestBodyProducerAborted() = 0; + /// whether it is not too late to write to the server + bool canSend(int fd) const; // sending of the request body to the server void sendMoreRequestBody(); // has body; kids overwrite to increment I/O stats counters diff -u -r -N squid-3.1.3/src/tests/stub_cache_cf.cc squid-3.1.4/src/tests/stub_cache_cf.cc --- squid-3.1.3/src/tests/stub_cache_cf.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/tests/stub_cache_cf.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,7 +1,7 @@ /* * $Id$ * - * DEBUG: section 3 Configuration File Parsing + * DEBUG: section 03 Configuration File Parsing * AUTHOR: Robert Collins * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/ufsdump.cc squid-3.1.4/src/ufsdump.cc --- squid-3.1.3/src/ufsdump.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/ufsdump.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,8 +1,7 @@ - /* * $Id$ * - * DEBUG: section 0 UFS Store Dump + * DEBUG: section 00 UFS Store Dump Tool * AUTHOR: Robert Collins * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/unlinkd.cc squid-3.1.4/src/unlinkd.cc --- squid-3.1.3/src/unlinkd.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/unlinkd.cc 2010-05-31 01:21:11.000000000 +1200 @@ -2,7 +2,7 @@ /* * $Id$ * - * DEBUG: section 2 Unlink Daemon + * DEBUG: section 02 Unlink Daemon * AUTHOR: Duane Wessels * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/unlinkd_daemon.cc squid-3.1.4/src/unlinkd_daemon.cc --- squid-3.1.3/src/unlinkd_daemon.cc 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/src/unlinkd_daemon.cc 2010-05-31 01:21:12.000000000 +1200 @@ -2,7 +2,7 @@ /* * $Id$ * - * DEBUG: - Unlink Daemon + * DEBUG: section -- Unlink Daemon * AUTHOR: Duane Wessels * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/src/wccp2.cc squid-3.1.4/src/wccp2.cc --- squid-3.1.3/src/wccp2.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/wccp2.cc 2010-05-31 01:21:11.000000000 +1200 @@ -2278,7 +2278,7 @@ port = strsep(&tmp2, ","); } - if (i == 8) { + if (i == WCCP2_NUMPORTS && port) { fatalf("parse_wccp2_service_ports: too many ports (maximum: 8) in list '%s'\n", options); } diff -u -r -N squid-3.1.3/src/wordlist.cc squid-3.1.4/src/wordlist.cc --- squid-3.1.3/src/wordlist.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/src/wordlist.cc 2010-05-31 01:21:11.000000000 +1200 @@ -2,7 +2,7 @@ /* * $Id$ * - * DEBUG: section 3 Configuration File Parsing + * DEBUG: section 03 Configuration File Parsing * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/test-suite/ESIExpressions.cc squid-3.1.4/test-suite/ESIExpressions.cc --- squid-3.1.3/test-suite/ESIExpressions.cc 2010-05-02 22:47:08.000000000 +1200 +++ squid-3.1.4/test-suite/ESIExpressions.cc 2010-05-31 01:21:11.000000000 +1200 @@ -2,7 +2,7 @@ /* * $Id$ * - * DEBUG: section 86 ESI Expressions + * DEBUG: section 86 ESI Expressions * AUTHOR: Robert Collins * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/test-suite/MemPoolTest.cc squid-3.1.4/test-suite/MemPoolTest.cc --- squid-3.1.3/test-suite/MemPoolTest.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/test-suite/MemPoolTest.cc 2010-05-31 01:21:12.000000000 +1200 @@ -34,7 +34,7 @@ #include "squid.h" -#if !DISABLE_POOLS +#if USE_CHUNKEDMEMPOOLS #include "MemPool.h" @@ -82,12 +82,12 @@ delete Pool; } -#endif /* DISABLE_POOLS */ +#endif /* USE_CHUNKEDMEMPOOLS */ int main (int argc, char **argv) { -#if !DISABLE_POOLS +#if USE_CHUNKEDMEMPOOLS MemPoolTest aTest; aTest.run(); #endif diff -u -r -N squid-3.1.3/test-suite/refcount.cc squid-3.1.4/test-suite/refcount.cc --- squid-3.1.3/test-suite/refcount.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/test-suite/refcount.cc 2010-05-31 01:21:12.000000000 +1200 @@ -2,7 +2,7 @@ /* * $Id$ * - * DEBUG: section xx Refcount allocator + * DEBUG: section -- Refcount allocator * AUTHOR: Robert Collins * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/tools/cachemgr.cc squid-3.1.4/tools/cachemgr.cc --- squid-3.1.3/tools/cachemgr.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/tools/cachemgr.cc 2010-05-31 01:21:11.000000000 +1200 @@ -1,5 +1,5 @@ /* - * DEBUG: section 0 CGI Cache Manager + * DEBUG: section -- CGI Cache Manager * AUTHOR: Duane Wessels * * SQUID Web Proxy Cache http://www.squid-cache.org/ diff -u -r -N squid-3.1.3/tools/squidclient.cc squid-3.1.4/tools/squidclient.cc --- squid-3.1.3/tools/squidclient.cc 2010-05-02 22:47:07.000000000 +1200 +++ squid-3.1.4/tools/squidclient.cc 2010-05-31 01:21:12.000000000 +1200 @@ -1,7 +1,7 @@ /* * $Id$ * - * DEBUG: section 0 WWW Client + * DEBUG: section -- WWW Client * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/