>From - Sun Jun 8 12:53:10 1997 Path: news.nacamar.de!newsfeed.nacamar.de!su-news-hub1.bbnplanet.com!cpk-news-hub1.bbnplanet.com!news.bbnplanet.com!news.spinne.com!not-for-mail From: Jeff Garzik Newsgroups: news.software.nntp Subject: INN pre-commit cache (patch) Date: Thu, 05 Jun 1997 00:39:09 -0400 Organization: Spinne http://spinne.com Lines: 293 Message-ID: <339642ED.340D5ADB@spinne.com> NNTP-Posting-Host: 206.240.111.163 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="------------240E22F7590929A01187AD2A" X-Mailer: Mozilla 3.01Gold (X11; I; Linux 2.0.30 i486) Xref: news.nacamar.de news.software.nntp:38904 This is a multi-part message in MIME format. --------------240E22F7590929A01187AD2A Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit This patch is a little bit more experimental than the previous history-cache patch. It adds a "pre-commit" cache, which locks a message id in a hash table for 30 seconds. All CHECK and IHAVE requests for that message id during those 30 seconds will be refused, [hopefully] saving bandwidth. Cache store collisions simply overwrite the existing cache entry. The logic is: IHAVE if ((msg id cached) AND (not timed out)) Refuse article otherwise (check history, ...) otherwise Put msg id in pre-commit cache (go to 'get article' state, ...) CHECK if ((msg id cached) AND (not timed out)) Refuse article otherwise (check history, ...) otherwise Put msg id in pre-commit cache (Reply with 'sendme' code, ...) TAKETHIS (msg id checks...) Put msg id in pre-commit cache (go to 'get article' state, ...) Like the previous patch, it applies to a clean inn-1.5.1 source tree. Note that this conflicts with apb's jumbo patch, because of his NCwritereply updates. The commented-out #define NCPRECOM_PEDANTIC will, if enabled, remove each msg id from the pre-commit cache after it is rejected or accepted by the system -- ie. after it's in the history cache. The default is just let the msg ids expire after 30 seconds. The commented-out #define NCPRECOM_DEFER will, if enabled, return a defer code to a streaming server upon a cache hit, instead of refusing the article. This is more proper, but could slow down a feed. The default is to refuse the msg id for the duration of its cache life. Creds to MD, JA, CO'N, and BB for ideas. Bug me about problems. -- Jeff Garzik News Administrator, Spinne, Inc. http://www.spinne.com/usenet/ --------------240E22F7590929A01187AD2A Content-Type: text/plain; charset=us-ascii; name="precommit-cache.patch" Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="precommit-cache.patch" *** ../inn-1.5.1/innd/nc.c Tue Dec 17 09:40:40 1996 --- innd/nc.c Wed Jun 4 23:44:23 1997 *************** *** 1,4 **** ! /* $Revision: 1.51 $ ** ** Routines for the NNTP channel. Other channels get the descriptors which ** we turn into NNTP channels, and over which we speak NNTP. --- 1,4 ---- ! /* $Revision: 1.51+precommit-cache$ ** ** Routines for the NNTP channel. Other channels get the descriptors which ** we turn into NNTP channels, and over which we speak NNTP. *************** *** 105,110 **** --- 105,149 ---- /* + * precommit cache + */ + #define NCPRECOMCACHESIZE 4096L + #define NCPRECOMCACHETIMEOUT 30 + + /* uncomment line below to: + * - zap cache entry after art acceptance or rejection + */ + /* #define NCPRECOM_PEDANTIC */ + + /* uncomment line below to: + * - have CHECK return DEFER instead of REFUSE + * for cache hits + */ + /* #define NCPRECOM_DEFER */ + + typedef struct { + long MessageIDHash; + time_t Timestamp; + } _NCprecomcache; + + STATIC _NCprecomcache NCprecomcache[NCPRECOMCACHESIZE]; + unsigned long NCprecomhits; + unsigned long NCprecomlookups; + STATIC long NCprecompos, NCprecomval; + STATIC NCprecominit = 0; + + + STATIC void + NCprecomclear () + { + memset((void *)&NCprecomcache, 0, sizeof(NCprecomcache)); + NCprecomhits = NCprecomlookups = 0; + } + + + + + /* ** Clear the work-in-progress entry and wake up anyone who might ** have been waiting for us. */ *************** *** 256,261 **** --- 295,307 ---- cp->Reported = 0; } NCwritereply(cp, response); + + #ifdef NCPRECOM_PEDANTIC + NCprecomval = dbzhash(wp->MessageID, strlen(wp->MessageID)); + NCprecompos = NCprecomval % NCPRECOMCACHESIZE; + NCprecomcache[NCprecompos].MessageIDHash = 0L; + #endif + cp->State = CSgetcmd; break; *************** *** 582,588 **** if (NCbadid(cp, p)) return; ! if (HIShavearticle(p)) { cp->Refused++; NCwritetext(cp, NNTP_HAVEIT); } --- 628,647 ---- if (NCbadid(cp, p)) return; ! /* compute msg id hash for pre-commit cache check */ ! NCprecomval = dbzhash(p, strlen(p)); ! NCprecompos = NCprecomval % NCPRECOMCACHESIZE; ! NCprecomlookups++; ! ! /* check pre-commit cache for msg id, with timeout */ ! if ((NCprecomcache[NCprecompos].MessageIDHash == NCprecomval) && ! ((NCprecomcache[NCprecompos].Timestamp + NCPRECOMCACHETIMEOUT) > Now.time)) { ! NCprecomhits++; ! cp->Refused++; ! NCwritetext(cp, NNTP_HAVEIT); ! } ! ! else if (HIShavearticle(p)) { cp->Refused++; NCwritetext(cp, NNTP_HAVEIT); } *************** *** 599,604 **** --- 658,667 ---- #endif /* defined(NNTP_RESENDIT_LATER) */ } else { + /* pre-commit msg id so that noone else offers us this for 30 seconds */ + NCprecomcache[NCprecompos].MessageIDHash = NCprecomval; + NCprecomcache[NCprecompos].Timestamp = Now.time; + NCwritetext(cp, NNTP_SENDIT); cp->State = CSgetarticle; } *************** *** 1387,1392 **** --- 1450,1463 ---- register CHANNEL *cp; int i; + /* one-time initialization of pre-commit cache */ + if (!NCprecominit) { + NCprecomclear(); + NCprecominit = 1; + syslog(L_NOTICE, "Pre-commit cache initialized: %ld entries, %ld bytes", + NCPRECOMCACHESIZE, (long) sizeof(NCprecomcache)); + } + /* Create the channel. */ cp = CHANcreate(fd, CTnntp, MustAuthorize ? CSgetauth : CSgetcmd, NCreader, NCwritedone); *************** *** 1486,1499 **** return; } ! if (HIShavearticle(p)) { cp->Refused++; (void)sprintf(cp->Sendid.Data, "%d %s", NNTP_ERR_GOTID_VAL, p); NCwritereply(cp, cp->Sendid.Data); } else if (NCinprogress(cp, p, &who)) { (void)sprintf(cp->Sendid.Data, "%d %s", NNTP_RESENDID_VAL, p); NCwritereply(cp, cp->Sendid.Data); } else { (void)sprintf(cp->Sendid.Data, "%d %s", NNTP_OK_SENDID_VAL, p); NCwritereply(cp, cp->Sendid.Data); } --- 1557,1598 ---- return; } ! /* compute msg id hash for pre-commit cache check */ ! NCprecomval = dbzhash(p, msglen - 5); ! NCprecompos = NCprecomval % NCPRECOMCACHESIZE; ! NCprecomlookups++; ! ! /* check pre-commit cache for msg id, with timeout */ ! if ((NCprecomcache[NCprecompos].MessageIDHash == NCprecomval) && ! ((NCprecomcache[NCprecompos].Timestamp + NCPRECOMCACHETIMEOUT) > Now.time)) { ! NCprecomhits++; ! ! #ifdef NCPRECOM_DEFER ! (void)sprintf(cp->Sendid.Data, "%d %s", NNTP_RESENDID_VAL, p); ! #else ! cp->Refused++; ! (void)sprintf(cp->Sendid.Data, "%d %s", NNTP_ERR_GOTID_VAL, p); ! #endif ! NCwritereply(cp, cp->Sendid.Data); ! } ! ! /* check history for msg id existence */ ! else if (HIShavearticle(p)) { cp->Refused++; (void)sprintf(cp->Sendid.Data, "%d %s", NNTP_ERR_GOTID_VAL, p); NCwritereply(cp, cp->Sendid.Data); + + /* check to see if article's already in progress on some other channel */ } else if (NCinprogress(cp, p, &who)) { (void)sprintf(cp->Sendid.Data, "%d %s", NNTP_RESENDID_VAL, p); NCwritereply(cp, cp->Sendid.Data); + + /* otherwise, we don't have the message, tell remote to send it to us */ } else { + /* pre-commit msg id so that noone else offers us this for 30 seconds */ + NCprecomcache[NCprecompos].MessageIDHash = NCprecomval; + NCprecomcache[NCprecompos].Timestamp = Now.time; + (void)sprintf(cp->Sendid.Data, "%d %s", NNTP_OK_SENDID_VAL, p); NCwritereply(cp, cp->Sendid.Data); } *************** *** 1518,1527 **** continue; for ( ; ISWHITE(*p); p++) continue; if (!ARTidok(p)) { syslog(L_NOTICE, "%s bad_messageid %s", CHANname(cp), MaxLength(p, p)); } ! msglen = strlen(p) + 5; /* 3 digits + space + id + null */ if (cp->Sendid.Size < msglen) { if (cp->Sendid.Size > 0) DISPOSE(cp->Sendid.Data); if (msglen > MAXHEADERSIZE) cp->Sendid.Size = msglen; --- 1617,1637 ---- continue; for ( ; ISWHITE(*p); p++) continue; + + msglen = strlen(p) + 5; /* 3 digits + space + id + null */ + if (!ARTidok(p)) { syslog(L_NOTICE, "%s bad_messageid %s", CHANname(cp), MaxLength(p, p)); } ! ! /* pre-commit msg id so that noone else offers us this for 30 seconds */ ! else { ! NCprecomval = dbzhash(p, msglen - 5); ! NCprecompos = NCprecomval % NCPRECOMCACHESIZE; ! NCprecomcache[NCprecompos].MessageIDHash = NCprecomval; ! NCprecomcache[NCprecompos].Timestamp = Now.time; ! } ! if (cp->Sendid.Size < msglen) { if (cp->Sendid.Size > 0) DISPOSE(cp->Sendid.Data); if (msglen > MAXHEADERSIZE) cp->Sendid.Size = msglen; --------------240E22F7590929A01187AD2A--