Index: dspam/CHANGE diff -c dspam/CHANGE:1.1.1.12 dspam/CHANGE:1.1.1.12.2.1 *** dspam/CHANGE:1.1.1.12 Sat Nov 8 13:55:54 2003 --- dspam/CHANGE Sat Nov 15 18:34:58 2003 *************** *** 1,3 **** --- 1,16 ---- + Version NEXT + ------------ + + [20031113.1938] jonz: corrected DSM_CLASSIFY bug in libdspam + + corrected two bugs in libdspam regarding the DSM_CLASSIFY mode: + + 1. CTX->signature would overwrite the provided signature with a new signature + resulting in a potential memory leak + + 2. If no signature was provided, DSM_CLASSIFY would segfault instead of create + a new signature + Version 2.8-beta-2 ------------------ Index: dspam/addspam.sh diff -c /dev/null dspam/addspam.sh:1.2.4.1 *** /dev/null Sat Nov 15 20:45:12 2003 --- dspam/addspam.sh Sat Nov 15 20:40:53 2003 *************** *** 0 **** --- 1,26 ---- + #!/bin/sh + + die() { + echo `date '+%b%d %H:%M:%S'` "$*" >&2 + exit 1 + } + + log() { + echo `date '+%b%d %H:%M:%S'` "$*" >&2 + } + + action="--`basename $0 .sh`" + log dspam -d $user $action + + exec >>/var/log/dspam.log 2>&1 + + read from || die "No input" + set - $from + envfrom="$2" + IFS="@" + set - $envfrom + user="$1" + domain="$2" + [ "$domain" = "yourcompany.com" ] || die "Invalid source domain: $domain" + log dspam -d $user $action + /usr/local/bin/dspam -d $user $action || die "DSPAM error" Index: dspam/decode.c diff -c dspam/decode.c:1.1.1.3 dspam/decode.c:1.1.1.3.2.1 *** dspam/decode.c:1.1.1.3 Thu Oct 23 17:25:44 2003 --- dspam/decode.c Sat Nov 15 18:34:58 2003 *************** *** 118,130 **** { /* If we see two boundaries converged on top of one another */ ! if (_ds_match_boundary (boundaries, line)) { /* Add the boundary as the terminating boundary */ current_block->terminating_boundary = strdup (line + 2); current_block->original_encoding = current_block->encoding; ! current_block = _ds_create_message_block (); ! if (current_block == NULL) { LOG (LOG_CRIT, ERROR_MEM_ALLOC); --- 118,131 ---- { /* If we see two boundaries converged on top of one another */ ! if (_ds_match_boundary (boundaries, line)) ! { /* Add the boundary as the terminating boundary */ current_block->terminating_boundary = strdup (line + 2); current_block->original_encoding = current_block->encoding; ! current_block = _ds_create_message_block (); ! if (current_block == NULL) { LOG (LOG_CRIT, ERROR_MEM_ALLOC); *************** *** 133,142 **** nt_destroy (boundaries); return NULL; } ! if (nt_add (out->components, (void *) current_block) == NULL) LOG (LOG_CRIT, ERROR_MEM_ALLOC); ! block_position = BP_HEADER; } --- 134,143 ---- nt_destroy (boundaries); return NULL; } ! if (nt_add (out->components, (void *) current_block) == NULL) LOG (LOG_CRIT, ERROR_MEM_ALLOC); ! block_position = BP_HEADER; } *************** *** 188,194 **** { /* Check for multipart boundary definition */ ! if (!strncasecmp (line, "Content-Type", 12) || ((line[0] == 32 || line[0] == 9) && in_content)) { char *h = strdup (line); --- 189,196 ---- { /* Check for multipart boundary definition */ ! if (!strncasecmp (line, "Content-Type", 12) ! || ((line[0] == 32 || line[0] == 9) && in_content)) { char *h = strdup (line); *************** *** 206,212 **** char *x; long pos, len; ! if (strchr(h, '=') && strchr(h, '"')) { x = strtok (strstr (h, "boundary"), "\""); x = strtok (NULL, "\""); --- 208,214 ---- char *x; long pos, len; ! if (strchr (h, '=') && strchr (h, '"')) { x = strtok (strstr (h, "boundary"), "\""); x = strtok (NULL, "\""); *************** *** 217,223 **** } /* Copy the case sensitive version back */ ! if (x != (char *) 1) { pos = x - h; len = strlen (x); memcpy (x, line + pos, len); --- 219,226 ---- } /* Copy the case sensitive version back */ ! if (x != (char *) 1) ! { pos = x - h; len = strlen (x); memcpy (x, line + pos, len); *************** *** 230,236 **** } } free (h); ! } else { in_content = 0; } } --- 233,241 ---- } } free (h); ! } ! else ! { in_content = 0; } } *************** *** 404,410 **** { LOG (LOG_CRIT, ERROR_MEM_ALLOC); } ! else { original = new_alloc; strcat (original, rest); --- 409,415 ---- { LOG (LOG_CRIT, ERROR_MEM_ALLOC); } ! else { original = new_alloc; strcat (original, rest); *************** *** 429,441 **** { LOG (LOG_CRIT, ERROR_MEM_ALLOC); } ! else { original = new_alloc; strcat (original, rest); } } ! free(data); data = original; } else --- 434,446 ---- { LOG (LOG_CRIT, ERROR_MEM_ALLOC); } ! else { original = new_alloc; strcat (original, rest); } } ! free (data); data = original; } else *************** *** 492,498 **** block->media_type = MT_MESSAGE; if (!strncasecmp (header->data + 8, "rfc822", 6)) block->media_subtype = MST_RFC822; ! else if (!strncasecmp(header->data + 8, "inoculation", 11)) block->media_subtype = MST_INOCULATION; else block->media_subtype = MST_OTHER; --- 497,503 ---- block->media_type = MT_MESSAGE; if (!strncasecmp (header->data + 8, "rfc822", 6)) block->media_subtype = MST_RFC822; ! else if (!strncasecmp (header->data + 8, "inoculation", 11)) block->media_subtype = MST_INOCULATION; else block->media_subtype = MST_OTHER; *************** *** 525,531 **** char *x; long pos, len; ! if (strchr (h, '=') && strchr(h, '"')) { x = strtok (strstr (h, "boundary"), "\""); x = strtok (NULL, "\""); --- 530,536 ---- char *x; long pos, len; ! if (strchr (h, '=') && strchr (h, '"')) { x = strtok (strstr (h, "boundary"), "\""); x = strtok (NULL, "\""); *************** *** 814,822 **** buffer_cat (out, "\n"); /* Assemble the bodies */ ! if (block->original_signed_body != NULL) buffer_cat (out, block->original_signed_body->data); ! else buffer_cat (out, block->body->data); if (block->terminating_boundary != NULL) --- 819,827 ---- buffer_cat (out, "\n"); /* Assemble the bodies */ ! if (block->original_signed_body != NULL) buffer_cat (out, block->original_signed_body->data); ! else buffer_cat (out, block->body->data); if (block->terminating_boundary != NULL) Index: dspam/dspam.c diff -c dspam/dspam.c:1.1.1.11 dspam/dspam.c:1.1.1.11.2.1 *** dspam/dspam.c:1.1.1.11 Mon Nov 3 17:41:19 2003 --- dspam/dspam.c Sat Nov 15 18:34:58 2003 *************** *** 63,69 **** { char mailer_args[256]; /* Args passed to MLOCAL */ char spam_args[256]; /* Args passed for spam delivery */ ! char managed_group[256]; /* Managed Group (if any) */ int mode = DSM_PROCESS; /* Processing mode */ #ifdef TRUSTED_USER_SECURITY int trusted_user = 0; /* Trusted User? */ --- 63,69 ---- { char mailer_args[256]; /* Args passed to MLOCAL */ char spam_args[256]; /* Args passed for spam delivery */ ! char managed_group[256]; /* Managed Group (if any) */ int mode = DSM_PROCESS; /* Processing mode */ #ifdef TRUSTED_USER_SECURITY int trusted_user = 0; /* Trusted User? */ *************** *** 81,97 **** char filename[MAX_FILENAME_LENGTH]; #ifdef DEBUG ! char debug_args[1024]; /* Report on all arguments passed in */ #endif ! int user_flag = 0; /* Next argument(s) are users */ ! int flags = 0; /* Agent (DAF) flags passed to process_message() */ ! int deliver_fp = 0; /* Deliver FPs? (Commandline) ! Only need to specify when configuring with ! --enable-spam-delivery */ ! int deliver_spam = 0; /* Deliver Spams? (Commandline) */ int i; int line = 1; --- 81,97 ---- char filename[MAX_FILENAME_LENGTH]; #ifdef DEBUG ! char debug_args[1024]; /* Report on all arguments passed in */ #endif ! int user_flag = 0; /* Next argument(s) are users */ ! int flags = 0; /* Agent (DAF) flags passed to process_message() */ ! int deliver_fp = 0; /* Deliver FPs? (Commandline) ! * Only need to specify when configuring with ! * --enable-spam-delivery */ ! int deliver_spam = 0; /* Deliver Spams? (Commandline) */ int i; int line = 1; *************** *** 140,148 **** if (!trusted_user) { ! LOG (LOG_INFO, ! "forcing username for untrusted user %s", ! p->pw_name); nt_add (users, p->pw_name); } #endif --- 140,146 ---- if (!trusted_user) { ! LOG (LOG_INFO, "forcing username for untrusted user %s", p->pw_name); nt_add (users, p->pw_name); } #endif *************** *** 213,221 **** if (!strcmp (argv[i], "--inoculate")) { ! if (!(flags & DAF_INOCULATE)) { #ifdef VERBOSE ! LOGDEBUG("setting flag DAF_INOCULATE"); #endif flags = flags ^ DAF_INOCULATE; } --- 211,220 ---- if (!strcmp (argv[i], "--inoculate")) { ! if (!(flags & DAF_INOCULATE)) ! { #ifdef VERBOSE ! LOGDEBUG ("setting flag DAF_INOCULATE"); #endif flags = flags ^ DAF_INOCULATE; } *************** *** 224,232 **** if (!strcmp (argv[i], "--corpus")) { ! if (!(flags & DAF_CORPUS)) { #ifdef VERBOSE ! LOGDEBUG("setting flag DAF_CORPUS"); #endif flags = flags ^ DAF_CORPUS; } --- 223,232 ---- if (!strcmp (argv[i], "--corpus")) { ! if (!(flags & DAF_CORPUS)) ! { #ifdef VERBOSE ! LOGDEBUG ("setting flag DAF_CORPUS"); #endif flags = flags ^ DAF_CORPUS; } *************** *** 239,245 **** continue; } ! /* backward compatibility */ if (!strcmp (argv[i], "--deliver-fp") || !strcmp (argv[i], "--deliver")) { deliver_fp = 1; --- 239,245 ---- continue; } ! /* backward compatibility */ if (!strcmp (argv[i], "--deliver-fp") || !strcmp (argv[i], "--deliver")) { deliver_fp = 1; *************** *** 253,259 **** continue; } ! if (!strcmp(argv[i], "--deliver-spam")) { deliver_spam = 1; continue; --- 253,259 ---- continue; } ! if (!strcmp (argv[i], "--deliver-spam")) { deliver_spam = 1; continue; *************** *** 325,331 **** /* Read in the message */ while ((fgets (buff, sizeof (buff), stdin)) != NULL) { ! if (line>1 || strncmp(buff, "From QUARANTINE", 15)) { if (buffer_cat (message, buff)) { LOG (LOG_CRIT, ERROR_MEM_ALLOC); --- 325,332 ---- /* Read in the message */ while ((fgets (buff, sizeof (buff), stdin)) != NULL) { ! if (line > 1 || strncmp (buff, "From QUARANTINE", 15)) ! { if (buffer_cat (message, buff)) { LOG (LOG_CRIT, ERROR_MEM_ALLOC); *************** *** 334,352 **** } /* Use the original user id if we are reversing a false positive */ ! if (mode == DSM_FALSEPOSITIVE && !strncasecmp(buff, "X-DSPAM-User: ", 14) && managed_group[0] != 0) { char user[MAX_USERNAME_LENGTH]; ! strlcpy(user, buff+14, sizeof(user)); ! chomp(user); ! LOGDEBUG("X-DSPAM-User: %s", user); ! nt_destroy(users); ! users = nt_create(NT_CHAR); ! if (users == NULL) { report_error (ERROR_MEM_ALLOC); exit (EXIT_FAILURE); } ! nt_add(users, user); } line++; --- 335,355 ---- } /* Use the original user id if we are reversing a false positive */ ! if (mode == DSM_FALSEPOSITIVE ! && !strncasecmp (buff, "X-DSPAM-User: ", 14) && managed_group[0] != 0) { char user[MAX_USERNAME_LENGTH]; ! strlcpy (user, buff + 14, sizeof (user)); ! chomp (user); ! LOGDEBUG ("X-DSPAM-User: %s", user); ! nt_destroy (users); ! users = nt_create (NT_CHAR); ! if (users == NULL) ! { report_error (ERROR_MEM_ALLOC); exit (EXIT_FAILURE); } ! nt_add (users, user); } line++; *************** *** 358,366 **** exit (EXIT_SUCCESS); } ! if (dspam_init_driver()) { ! LOG(LOG_WARNING, "unable to initialize storage driver"); ! exit(EXIT_FAILURE); } /* Process message for each user */ --- 361,370 ---- exit (EXIT_SUCCESS); } ! if (dspam_init_driver ()) ! { ! LOG (LOG_WARNING, "unable to initialize storage driver"); ! exit (EXIT_FAILURE); } /* Process message for each user */ *************** *** 387,425 **** if (!stat (filename, &s)) #endif { ! if (!(flags & DAF_CORPUS)) { ! retcode = deliver_message (message->data, (flags & DAF_STDOUT) ? NULL : mailer_args, node_nt->ptr, mode); ! if (retcode && exitcode == EXIT_SUCCESS) exitcode = retcode; } } else { ! result = process_message (managed_group, sizeof(managed_group), message, node_nt->ptr, mode, flags); ! if (mode == DSM_CLASSIFY) { ! if (result == DSR_ISSPAM) ! printf("SPAM\n"); else ! printf("HAM\n"); node_nt = c_nt_next (users, &c_nt); continue; } if (result != DSR_ISSPAM) { ! if (result != DSR_ISINNOCENT) { LOG (LOG_WARNING, "process_message returned error %d. delivering message.", result); } #ifdef DELIVER_SPAM ! if (mode != DSM_FALSEPOSITIVE && !deliver_fp) { #endif LOGDEBUG ("delivering message"); ! if (!(flags & DAF_CORPUS)) { ! retcode = deliver_message (message->data, (flags & DAF_STDOUT) ? NULL : mailer_args, node_nt->ptr, mode); if (retcode) exitcode = retcode; } --- 391,442 ---- if (!stat (filename, &s)) #endif { ! if (!(flags & DAF_CORPUS)) ! { ! retcode = ! deliver_message (message->data, ! (flags & DAF_STDOUT) ! ? NULL : mailer_args, node_nt->ptr, mode); ! if (retcode && exitcode == EXIT_SUCCESS) exitcode = retcode; } } else { ! result = process_message (managed_group, ! sizeof (managed_group), ! message, node_nt->ptr, mode, flags); ! if (mode == DSM_CLASSIFY) ! { ! if (result == DSR_ISSPAM) ! printf ("SPAM\n"); else ! printf ("HAM\n"); node_nt = c_nt_next (users, &c_nt); continue; } if (result != DSR_ISSPAM) { ! if (result != DSR_ISINNOCENT) ! { LOG (LOG_WARNING, "process_message returned error %d. delivering message.", result); } #ifdef DELIVER_SPAM ! if (mode != DSM_FALSEPOSITIVE && !deliver_fp) ! { #endif LOGDEBUG ("delivering message"); ! if (!(flags & DAF_CORPUS)) ! { ! retcode = ! deliver_message ! (message->data, ! (flags & DAF_STDOUT) ? NULL : mailer_args, node_nt->ptr, mode); if (retcode) exitcode = retcode; } *************** *** 434,460 **** { if (spam_args[0] != 0) { ! retcode = deliver_message (message->data, (flags & DAF_STDOUT) ? NULL : spam_args, node_nt->ptr, mode); if (retcode) exitcode = retcode; } else { ! if (!(flags & DAF_CORPUS)) { #ifdef DELIVER_SPAM ! retcode = deliver_message ! (message->data, (flags & DAF_STDOUT) ? NULL : mailer_args, node_nt->ptr, mode); if (retcode) exitcode = retcode; #else ! if (deliver_spam) { ! retcode = deliver_message (message->data, (flags & DAF_STDOUT) ? NULL : mailer_args, node_nt->ptr, mode); if (retcode) exitcode = retcode; ! } else { if (managed_group[0] == 0) quarantine_message (message->data, node_nt->ptr); ! else quarantine_message (message->data, managed_group); } #endif --- 451,493 ---- { if (spam_args[0] != 0) { ! retcode = ! deliver_message ! (message->data, ! (flags & DAF_STDOUT) ? NULL : spam_args, node_nt->ptr, mode); if (retcode) exitcode = retcode; } else { ! if (!(flags & DAF_CORPUS)) ! { #ifdef DELIVER_SPAM ! retcode = ! deliver_message ! (message-> ! data, ! (flags & ! DAF_STDOUT) ? NULL : mailer_args, node_nt->ptr, mode); if (retcode) exitcode = retcode; #else ! if (deliver_spam) ! { ! retcode = ! deliver_message ! (message-> ! data, ! (flags ! & DAF_STDOUT) ? NULL : mailer_args, node_nt->ptr, mode); if (retcode) exitcode = retcode; ! } ! else ! { if (managed_group[0] == 0) quarantine_message (message->data, node_nt->ptr); ! else quarantine_message (message->data, managed_group); } #endif *************** *** 466,472 **** node_nt = c_nt_next (users, &c_nt); } ! dspam_shutdown_driver(); LOGDEBUG ("DSPAM Agent Shutdown. Exit Code: %d", exitcode); exit (exitcode); } --- 499,505 ---- node_nt = c_nt_next (users, &c_nt); } ! dspam_shutdown_driver (); LOGDEBUG ("DSPAM Agent Shutdown. Exit Code: %d", exitcode); exit (exitcode); } *************** *** 480,490 **** */ int ! process_message (char *r_group, size_t size, buffer * message, const char *username, int mode, int flags) { ! DSPAM_CTX *CTX; /* dspam context */ struct _ds_message *components; ! struct _ds_spam_signature SIG; /* signature object */ char ctx_group[128]; struct nt_node *node_nt; --- 513,524 ---- */ int ! process_message (char *r_group, size_t size, buffer * message, ! const char *username, int mode, int flags) { ! DSPAM_CTX *CTX; /* dspam context */ struct _ds_message *components; ! struct _ds_spam_signature SIG; /* signature object */ char ctx_group[128]; struct nt_node *node_nt; *************** *** 501,515 **** int result; int i = 0; ! struct nt *inoc_users; /* inoculate list */ ! struct nt *classify_users; /* classify list */ ctx_group[0] = 0; #ifdef DEBUG { FILE *f; char m[MAX_FILENAME_LENGTH]; ! snprintf(m, sizeof(m), "%s/dspam.messages", USERDIR); f = fopen (m, "a"); if (f != NULL) { --- 535,549 ---- int result; int i = 0; ! struct nt *inoc_users; /* inoculate list */ ! struct nt *classify_users; /* classify list */ ctx_group[0] = 0; #ifdef DEBUG { FILE *f; char m[MAX_FILENAME_LENGTH]; ! snprintf (m, sizeof (m), "%s/dspam.messages", USERDIR); f = fopen (m, "a"); if (f != NULL) { *************** *** 519,533 **** } #endif ! inoc_users = nt_create(NT_CHAR); ! if (inoc_users == NULL) { ! LOG(LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } ! classify_users = nt_create(NT_CHAR); ! if (classify_users == NULL) { ! LOG(LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } --- 553,569 ---- } #endif ! inoc_users = nt_create (NT_CHAR); ! if (inoc_users == NULL) ! { ! LOG (LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } ! classify_users = nt_create (NT_CHAR); ! if (classify_users == NULL) ! { ! LOG (LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } *************** *** 544,557 **** int do_inoculate = 0; char *type, *list; chomp (buffer); ! list = strdup(buffer); group = strtok (buffer, ":"); if (group != NULL) { ! type = strtok(NULL, ":"); user = strtok (NULL, ","); ! ! if (!strcasecmp(type, "INOCULATION") && mode == DSM_ADDSPAM && (!(flags & DAF_CORPUS) || flags & DAF_INOCULATE)) do_inoculate = 1; while (user != NULL) --- 580,595 ---- int do_inoculate = 0; char *type, *list; chomp (buffer); ! list = strdup (buffer); group = strtok (buffer, ":"); if (group != NULL) { ! type = strtok (NULL, ":"); user = strtok (NULL, ","); ! ! if (!strcasecmp (type, "INOCULATION") ! && mode == DSM_ADDSPAM ! && (!(flags & DAF_CORPUS) || flags & DAF_INOCULATE)) do_inoculate = 1; while (user != NULL) *************** *** 560,605 **** { /* If we're reporting a spam, report it as a spam to all other ! users in the inoculation group */ ! if (do_inoculate) { char *l = list, *u; ! u = strsep(&l, ":"); ! u = strsep(&l, ":"); ! u = strsep(&l, ","); ! while(u != NULL) { ! if (strcmp(u, username)) { ! LOGDEBUG("adding user %s to inoculation group %s", u, group); ! nt_add(inoc_users, u); } ! u = strsep(&l, ","); } ! } else if (!strncasecmp(type, "SHARED", 6)) { strlcpy (ctx_group, group, sizeof (ctx_group)); LOGDEBUG ("assigning user %s to group %s", username, group); ! if (!strncasecmp(type+6, ",MANAGED", 8)) strlcpy (r_group, ctx_group, size); ! ! } else if (!strcasecmp(type, "CLASSIFICATION")) { char *l = list, *u; ! u = strsep(&l, ":"); ! u = strsep(&l, ":"); ! u = strsep(&l, ","); ! while(u != NULL) { ! if (strcmp(u, username)) { ! LOGDEBUG("adding user %s to classification group %s", u, group); ! nt_add(classify_users, u); } ! u = strsep(&l, ","); } ! } } user = strtok (NULL, ","); } } ! free(list); } fclose (file); } --- 598,654 ---- { /* If we're reporting a spam, report it as a spam to all other ! * users in the inoculation group */ ! if (do_inoculate) ! { char *l = list, *u; ! u = strsep (&l, ":"); ! u = strsep (&l, ":"); ! u = strsep (&l, ","); ! while (u != NULL) ! { ! if (strcmp (u, username)) ! { ! LOGDEBUG ("adding user %s to inoculation group %s", u, ! group); ! nt_add (inoc_users, u); } ! u = strsep (&l, ","); } ! } ! else if (!strncasecmp (type, "SHARED", 6)) ! { strlcpy (ctx_group, group, sizeof (ctx_group)); LOGDEBUG ("assigning user %s to group %s", username, group); ! if (!strncasecmp (type + 6, ",MANAGED", 8)) strlcpy (r_group, ctx_group, size); ! ! } ! else if (!strcasecmp (type, "CLASSIFICATION")) ! { char *l = list, *u; ! u = strsep (&l, ":"); ! u = strsep (&l, ":"); ! u = strsep (&l, ","); ! while (u != NULL) ! { ! if (strcmp (u, username)) ! { ! LOGDEBUG ("adding user %s to classification group %s", u, ! group); ! nt_add (classify_users, u); } ! u = strsep (&l, ","); } ! } } user = strtok (NULL, ","); } } ! free (list); } fclose (file); } *************** *** 614,620 **** CTX = dspam_init (username, ctx_group, mode, DSF_CHAINED | DSF_SIGNATURE | f_corpus | f_inoculate); #else ! CTX = dspam_init (username, ctx_group, mode, DSF_SIGNATURE | f_corpus | f_inoculate); #endif --- 663,669 ---- CTX = dspam_init (username, ctx_group, mode, DSF_CHAINED | DSF_SIGNATURE | f_corpus | f_inoculate); #else ! CTX = dspam_init (username, ctx_group, mode, DSF_SIGNATURE | f_corpus | f_inoculate); #endif *************** *** 636,650 **** /* If this message is an inoculation, authenticate and process it */ /* per Internet-Draft draft-spamfilt-inoculation-01.txt ! "The Versatile Language Inoculation Message Format" ! ! UNDER DEVELOPMENT... ! current caveats: ! ! - only md5 authentication is supported ! - only the message/inoculation MIME type is supported ! - outgoing inoculation messages are not supported ! */ #ifdef EXPERIMENTAL --- 685,699 ---- /* If this message is an inoculation, authenticate and process it */ /* per Internet-Draft draft-spamfilt-inoculation-01.txt ! * "The Versatile Language Inoculation Message Format" ! * ! * UNDER DEVELOPMENT... ! * current caveats: ! * ! * - only md5 authentication is supported ! * - only the message/inoculation MIME type is supported ! * - outgoing inoculation messages are not supported ! */ #ifdef EXPERIMENTAL *************** *** 655,773 **** int auth_user = 0, auth_checksum = 0; char *inoc_message = NULL; FILE *file; ! ! node_nt = c_nt_first(CTX->message->components, &c_nt); ! if (node_nt != NULL) { struct _ds_message_block *block; block = (struct _ds_message_block *) node_nt->ptr; ! if (block->media_type == MT_MESSAGE && block->media_subtype == MST_INOCULATION) { struct _ds_header_field *header; char *sender = NULL, *checksum = NULL, *inoculation_type = NULL; int auth_type = IAT_UNKNOWN; long content_length = 0; ! LOGDEBUG("validating inoculation"); /* Read the inoculation headers */ ! node_hnt = c_nt_first(block->headers, &c_nt); ! while(node_hnt != NULL) { header = (struct _ds_header_field *) node_hnt->ptr; ! if (header != NULL) { ! if (!strcasecmp(header->heading, "Content-Length")) ! content_length = strtol(header->data, NULL, 0); ! else if (!strcasecmp(header->heading, "Inoculation-Sender")) sender = header->data; ! else if (!strcasecmp(header->heading, "Inoculation-Authentication")) { ! if (!strncasecmp(header->data, "md5", 3)) { ! char *d = strdup(header->data); auth_type = IAT_MD5; ! if (strchr(header->data, '"')) { ! checksum = strtok(d, "\""); ! checksum = strdup(strtok(NULL, "\"")); ! } else { ! checksum = strtok(d, "="); ! checksum = strdup(strtok(NULL, "=")); } ! free(d); ! } else if (!strncasecmp(header->data, "signed", 6)) { auth_type = IAT_SIGNED; ! } else if (!strncasecmp(header->data, "none", 4)) { auth_type = IAT_NONE; } } ! else if (!strcasecmp(header->heading, "Inoculation-Type")) inoculation_type = header->data; } ! node_hnt = c_nt_next(block->headers, &c_nt); } /* Verify the inoculation */ ! if (sender != NULL && checksum != NULL && inoculation_type != NULL) { ! char *s = NULL, *v = NULL; /* s = sender, v = validation code */ ! /* Verify the sender, get the validation code */ ! strcpy(filename, _ds_userdir_path(username, "inoc")); ! file = fopen(filename, "r"); ! if (file != NULL) { char buf[1024]; ! while(fgets(buf, sizeof(buf), file) != NULL) { ! s = strdup(strtok(buf, ":")); ! v = strdup(strtok(NULL, ":")); ! if (!strcmp(s, sender)) { ! LOGDEBUG("sender verified"); ! auth_user = 1; ! } else { ! free(s); ! free(v); } } ! fclose(file); } /* Verify the Checksum */ ! if (auth_user) { ! if (auth_type == IAT_MD5) { unsigned char digest[17]; char a_digest[33]; char x[3]; MD5_CTX MTX; if (content_length == 0) ! content_length = strlen(block->body->data)-1; ! MD5Init(&MTX); ! MD5Update(&MTX, v, strlen(v)); /* Verification Code */ ! MD5Update(&MTX, block->body->data, content_length); ! MD5Final(digest, &MTX); a_digest[0] = 0; ! for(i=0;i<16;i++) { ! snprintf(x, 3, "%02x", digest[i]); ! strcat(a_digest, x); } ! if (!strcmp(a_digest, checksum)) { ! LOGDEBUG("inoculation validated"); auth_checksum = 1; ! inoc_message = strdup(block->body->data); ! } else { ! LOGDEBUG("inoculation invalid. ignoring"); return 0; } } } } ! free(checksum); } } /* Process the inoculation */ ! if (auth_user && auth_checksum) { ! inoculate_user(username, NULL, inoc_message); ! free(inoc_message); return DSR_ISSPAM; } } --- 704,849 ---- int auth_user = 0, auth_checksum = 0; char *inoc_message = NULL; FILE *file; ! ! node_nt = c_nt_first (CTX->message->components, &c_nt); ! if (node_nt != NULL) ! { struct _ds_message_block *block; block = (struct _ds_message_block *) node_nt->ptr; ! if (block->media_type == MT_MESSAGE ! && block->media_subtype == MST_INOCULATION) ! { struct _ds_header_field *header; char *sender = NULL, *checksum = NULL, *inoculation_type = NULL; int auth_type = IAT_UNKNOWN; long content_length = 0; ! LOGDEBUG ("validating inoculation"); /* Read the inoculation headers */ ! node_hnt = c_nt_first (block->headers, &c_nt); ! while (node_hnt != NULL) ! { header = (struct _ds_header_field *) node_hnt->ptr; ! if (header != NULL) ! { ! if (!strcasecmp (header->heading, "Content-Length")) ! content_length = strtol (header->data, NULL, 0); ! else if (!strcasecmp (header->heading, "Inoculation-Sender")) sender = header->data; ! else if (!strcasecmp ! (header->heading, "Inoculation-Authentication")) { ! if (!strncasecmp (header->data, "md5", 3)) ! { ! char *d = strdup (header->data); auth_type = IAT_MD5; ! if (strchr (header->data, '"')) ! { ! checksum = strtok (d, "\""); ! checksum = strdup (strtok (NULL, "\"")); } ! else ! { ! checksum = strtok (d, "="); ! checksum = strdup (strtok (NULL, "=")); ! } ! free (d); ! } ! else if (!strncasecmp (header->data, "signed", 6)) ! { auth_type = IAT_SIGNED; ! } ! else if (!strncasecmp (header->data, "none", 4)) ! { auth_type = IAT_NONE; } } ! else if (!strcasecmp (header->heading, "Inoculation-Type")) inoculation_type = header->data; } ! node_hnt = c_nt_next (block->headers, &c_nt); } /* Verify the inoculation */ ! if (sender != NULL && checksum != NULL && inoculation_type != NULL) ! { ! char *s = NULL, *v = NULL; /* s = sender, v = validation code */ ! /* Verify the sender, get the validation code */ ! strcpy (filename, _ds_userdir_path (username, "inoc")); ! file = fopen (filename, "r"); ! if (file != NULL) ! { char buf[1024]; ! while (fgets (buf, sizeof (buf), file) != NULL) ! { ! s = strdup (strtok (buf, ":")); ! v = strdup (strtok (NULL, ":")); ! if (!strcmp (s, sender)) ! { ! LOGDEBUG ("sender verified"); ! auth_user = 1; ! } ! else ! { ! free (s); ! free (v); } } ! fclose (file); } /* Verify the Checksum */ ! if (auth_user) ! { ! if (auth_type == IAT_MD5) ! { unsigned char digest[17]; char a_digest[33]; char x[3]; MD5_CTX MTX; if (content_length == 0) ! content_length = strlen (block->body->data) - 1; ! MD5Init (&MTX); ! MD5Update (&MTX, v, strlen (v)); /* Verification Code */ ! MD5Update (&MTX, block->body->data, content_length); ! MD5Final (digest, &MTX); a_digest[0] = 0; ! for (i = 0; i < 16; i++) ! { ! snprintf (x, 3, "%02x", digest[i]); ! strcat (a_digest, x); } ! if (!strcmp (a_digest, checksum)) ! { ! LOGDEBUG ("inoculation validated"); auth_checksum = 1; ! inoc_message = strdup (block->body->data); ! } ! else ! { ! LOGDEBUG ("inoculation invalid. ignoring"); return 0; } } } } ! free (checksum); } } /* Process the inoculation */ ! if (auth_user && auth_checksum) ! { ! inoculate_user (username, NULL, inoc_message); ! free (inoc_message); return DSR_ISSPAM; } } *************** *** 956,963 **** while (node_hnt != NULL) { field = (struct _ds_header_field *) node_hnt->ptr; ! if (field != NULL && field->heading != NULL ! && field->data != NULL) if (!strncasecmp (field->heading, "Content-Disposition", 19)) if (!strncasecmp (field->data, "attachment", 10)) is_attachment = 1; --- 1032,1039 ---- while (node_hnt != NULL) { field = (struct _ds_header_field *) node_hnt->ptr; ! if (field != NULL ! && field->heading != NULL && field->data != NULL) if (!strncasecmp (field->heading, "Content-Disposition", 19)) if (!strncasecmp (field->data, "attachment", 10)) is_attachment = 1; *************** *** 987,994 **** node_header = c_nt_first (block->headers, &c2); while (node_header != NULL) { ! struct _ds_header_field ! *header = (struct _ds_header_field *) node_header->ptr; if (!strcasecmp (header->heading, "Content-Transfer-Encoding")) { --- 1063,1070 ---- node_header = c_nt_first (block->headers, &c2); while (node_header != NULL) { ! struct _ds_header_field *header = ! (struct _ds_header_field *) node_header->ptr; if (!strcasecmp (header->heading, "Content-Transfer-Encoding")) { *************** *** 1071,1088 **** } } ! if (!have_signature && mode != DSM_PROCESS) { CTX->flags = CTX->flags ^ DSF_SIGNATURE; CTX->signature = NULL; } ! if (have_signature && mode != DSM_PROCESS) { #ifdef TEST_COND_TRAINING int do_train = 1, iter = 0, ck_result = 0; /* train until test conditions are met, 5 iterations max */ ! while(do_train && iter < 5) { DSPAM_CTX *CLX; int match = (CTX->mode == DSM_ADDSPAM) ? DSR_ISSPAM : DSR_ISINNOCENT; --- 1147,1166 ---- } } ! if (!have_signature && mode != DSM_PROCESS) { CTX->flags = CTX->flags ^ DSF_SIGNATURE; CTX->signature = NULL; } ! if (have_signature && mode != DSM_PROCESS) ! { #ifdef TEST_COND_TRAINING int do_train = 1, iter = 0, ck_result = 0; /* train until test conditions are met, 5 iterations max */ ! while (do_train && iter < 5) ! { DSPAM_CTX *CLX; int match = (CTX->mode == DSM_ADDSPAM) ? DSR_ISSPAM : DSR_ISINNOCENT; *************** *** 1092,1131 **** result = dspam_process (CTX, NULL); #ifdef TEST_COND_TRAINING ! if (iter > 1) { ! if (match == DSR_ISSPAM) { CTX->totals.spam_misclassified--; CTX->totals.total_spam--; ! } else { CTX->totals.innocent_misclassified--; CTX->totals.total_innocent--; } } ! LOGDEBUG("reclassifying iteration %d result: %d", iter, result); /* only attempt test-conditional training on a mature corpus */ if (CTX->totals.total_innocent < 4000 && CTX->mode == DSM_ADDSPAM) do_train = 0; ! else { /* CLX = Classify Context */ #ifdef CHAINED_TOKENS ! CLX = dspam_init (CTX->username, CTX->group, DSM_CLASSIFY, ! DSF_CHAINED | DSF_SIGNATURE); #else ! CLX = dspam_init (CTX->username, CTX->group, DSM_CLASSIFY, ! DSF_SIGNATURE); #endif ! if (!CLX) { do_train = 0; break; } CLX->signature = &SIG; ck_result = dspam_process (CLX, NULL); ! if (ck_result || CLX->result == match) do_train = 0; CLX->signature = NULL; ! dspam_destroy(CLX); } } #endif --- 1170,1215 ---- result = dspam_process (CTX, NULL); #ifdef TEST_COND_TRAINING ! if (iter > 1) ! { ! if (match == DSR_ISSPAM) ! { CTX->totals.spam_misclassified--; CTX->totals.total_spam--; ! } ! else ! { CTX->totals.innocent_misclassified--; CTX->totals.total_innocent--; } } ! LOGDEBUG ("reclassifying iteration %d result: %d", iter, result); /* only attempt test-conditional training on a mature corpus */ if (CTX->totals.total_innocent < 4000 && CTX->mode == DSM_ADDSPAM) do_train = 0; ! else ! { /* CLX = Classify Context */ #ifdef CHAINED_TOKENS ! CLX = dspam_init (CTX->username, CTX->group, ! DSM_CLASSIFY, DSF_CHAINED | DSF_SIGNATURE); #else ! CLX = dspam_init (CTX->username, CTX->group, ! DSM_CLASSIFY, DSF_SIGNATURE); #endif ! if (!CLX) ! { do_train = 0; break; } CLX->signature = &SIG; ck_result = dspam_process (CLX, NULL); ! if (ck_result || CLX->result == match) do_train = 0; CLX->signature = NULL; ! dspam_destroy (CLX); } } #endif *************** *** 1140,1164 **** result = CTX->result; ! if (result != DSR_ISSPAM && mode == DSM_PROCESS && CTX->confidence < 0.65) { struct nt_node *node_int; struct nt_c c_i; - - node_int = c_nt_first(classify_users, &c_i); - while(node_int != NULL && result != DSR_ISSPAM) { - LOGDEBUG("checking result for user %s", (const char *) node_int->ptr); - result = user_classify((const char *) node_int->ptr, CTX->signature, NULL); - if (result == DSR_ISSPAM) { - LOGDEBUG("CLASSIFY CATCH: %s", (const char *) node_int->ptr); - CTX->result = result; - } ! node_int = c_nt_next(classify_users, &c_i); } /* Add as spam */ ! if (result == DSR_ISSPAM) { CTX->mode = DSM_ADDSPAM; dspam_process (CTX, NULL); CTX->totals.spam_misclassified--; --- 1224,1253 ---- result = CTX->result; ! if (result != DSR_ISSPAM && mode == DSM_PROCESS && CTX->confidence < 0.65) ! { struct nt_node *node_int; struct nt_c c_i; ! node_int = c_nt_first (classify_users, &c_i); ! while (node_int != NULL && result != DSR_ISSPAM) ! { ! LOGDEBUG ("checking result for user %s", (const char *) node_int->ptr); ! result = user_classify ((const char *) node_int->ptr, ! CTX->signature, NULL); ! if (result == DSR_ISSPAM) ! { ! LOGDEBUG ("CLASSIFY CATCH: %s", (const char *) node_int->ptr); ! CTX->result = result; ! } ! ! node_int = c_nt_next (classify_users, &c_i); } /* Add as spam */ ! if (result == DSR_ISSPAM) ! { CTX->mode = DSM_ADDSPAM; dspam_process (CTX, NULL); CTX->totals.spam_misclassified--; *************** *** 1167,1193 **** } /* Inoculate other users: Signature */ ! if (have_signature && mode == DSM_ADDSPAM && inoc_users->items>0) { struct nt_node *node_int; ! struct nt_c c_i; ! node_int = c_nt_first(inoc_users, &c_i); ! while(node_int != NULL) { ! inoculate_user((const char *) node_int->ptr, &SIG, NULL); ! node_int = c_nt_next(inoc_users, &c_i); } } /* Inoculate other users: Corpus */ ! if (!have_signature && mode == DSM_ADDSPAM && flags & DAF_INOCULATE && inoc_users->items>0) { struct nt_node *node_int; struct nt_c c_i; ! node_int = c_nt_first(inoc_users, &c_i); ! while(node_int != NULL) { ! inoculate_user((const char *) node_int->ptr, NULL, message->data); ! node_int = c_nt_next(inoc_users, &c_i); } ! inoculate_user(username, NULL, message->data); return DSR_ISSPAM; } --- 1256,1287 ---- } /* Inoculate other users: Signature */ ! if (have_signature && mode == DSM_ADDSPAM && inoc_users->items > 0) ! { struct nt_node *node_int; ! struct nt_c c_i; ! node_int = c_nt_first (inoc_users, &c_i); ! while (node_int != NULL) ! { ! inoculate_user ((const char *) node_int->ptr, &SIG, NULL); ! node_int = c_nt_next (inoc_users, &c_i); } } /* Inoculate other users: Corpus */ ! if (!have_signature && mode == DSM_ADDSPAM && flags & DAF_INOCULATE ! && inoc_users->items > 0) ! { struct nt_node *node_int; struct nt_c c_i; ! node_int = c_nt_first (inoc_users, &c_i); ! while (node_int != NULL) ! { ! inoculate_user ((const char *) node_int->ptr, NULL, message->data); ! node_int = c_nt_next (inoc_users, &c_i); } ! inoculate_user (username, NULL, message->data); return DSR_ISSPAM; } *************** *** 1205,1211 **** valid = 1; } ! LOGDEBUG("saved signature as %s", session); #ifdef SIGNATURE_ATTACHMENTS --- 1299,1305 ---- valid = 1; } ! LOGDEBUG ("saved signature as %s", session); #ifdef SIGNATURE_ATTACHMENTS *************** *** 1632,1638 **** #endif } ! write_web_stats ((CTX->group == NULL) ? username : CTX->group, &CTX->totals); LOGDEBUG ("libdspam returned probability of %f, message result: %s", CTX->probability, (result != DSR_ISSPAM) ? "NOT SPAM" : "SPAM"); --- 1726,1733 ---- #endif } ! write_web_stats ((CTX->group == NULL) ? username : CTX->group, ! &CTX->totals); LOGDEBUG ("libdspam returned probability of %f, message result: %s", CTX->probability, (result != DSR_ISSPAM) ? "NOT SPAM" : "SPAM"); *************** *** 1693,1699 **** else LOG (LOG_CRIT, ERROR_MEM_ALLOC); ! if (result == DSR_ISSPAM) { head = malloc (sizeof (struct _ds_header_field)); if (head != NULL) { --- 1788,1795 ---- else LOG (LOG_CRIT, ERROR_MEM_ALLOC); ! if (result == DSR_ISSPAM) ! { head = malloc (sizeof (struct _ds_header_field)); if (head != NULL) { *************** *** 1746,1755 **** { block = (struct _ds_message_block *) node_nt->ptr; ! if (block-> ! terminating_boundary != ! NULL && !strcmp (block->terminating_boundary, term) ! && term[0] != 0) { struct nt_node *old; free (block->terminating_boundary); --- 1842,1849 ---- { block = (struct _ds_message_block *) node_nt->ptr; ! if (block->terminating_boundary != NULL ! && !strcmp (block->terminating_boundary, term) && term[0] != 0) { struct nt_node *old; free (block->terminating_boundary); *************** *** 1777,1784 **** char term[512]; char scratch[128]; ! snprintf (scratch, sizeof (scratch), "%s%s%s\n", SIGNATURE_BEGIN, ! session, SIGNATURE_END); /* Create new message block */ newblock = --- 1871,1878 ---- char term[512]; char scratch[128]; ! snprintf (scratch, sizeof (scratch), ! "%s%s%s\n", SIGNATURE_BEGIN, session, SIGNATURE_END); /* Create new message block */ newblock = *************** *** 1809,1816 **** newblock->body = buffer_create (scratch); newblock->original_signed_body = NULL; ! field = ! (struct _ds_header_field *) malloc (sizeof (struct _ds_header_field)); if (field == NULL) { --- 1903,1909 ---- newblock->body = buffer_create (scratch); newblock->original_signed_body = NULL; ! field = (struct _ds_header_field *) malloc (sizeof (struct _ds_header_field)); if (field == NULL) { *************** *** 1822,1829 **** field->data = strdup ("text/plain; name=\"dspam.txt\""); nt_add (newblock->headers, field); ! field = ! (struct _ds_header_field *) malloc (sizeof (struct _ds_header_field)); if (field == NULL) { --- 1915,1921 ---- field->data = strdup ("text/plain; name=\"dspam.txt\""); nt_add (newblock->headers, field); ! field = (struct _ds_header_field *) malloc (sizeof (struct _ds_header_field)); if (field == NULL) { *************** *** 1944,1950 **** { char ip[32]; char mta_whitelist[MAX_FILENAME_LENGTH]; ! snprintf(mta_whitelist, MAX_FILENAME_LENGTH, "%s/mta.whitelist", USERDIR); if (!dspam_getsource (CTX, ip, sizeof (ip), mta_whitelist)) { --- 2036,2043 ---- { char ip[32]; char mta_whitelist[MAX_FILENAME_LENGTH]; ! snprintf (mta_whitelist, MAX_FILENAME_LENGTH, ! "%s/mta.whitelist", USERDIR); if (!dspam_getsource (CTX, ip, sizeof (ip), mta_whitelist)) { *************** *** 1964,1971 **** _ds_destroy_message (CTX->message); dspam_destroy (CTX); ! nt_destroy(inoc_users); ! nt_destroy(classify_users); return result; } --- 2057,2064 ---- _ds_destroy_message (CTX->message); dspam_destroy (CTX); ! nt_destroy (inoc_users); ! nt_destroy (classify_users); return result; } *************** *** 1981,1988 **** FILE *file; int rc; ! if (mailer_args == NULL) { ! fputs(message, stdout); return 0; } --- 2074,2082 ---- FILE *file; int rc; ! if (mailer_args == NULL) ! { ! fputs (message, stdout); return 0; } *************** *** 2057,2071 **** char filename[MAX_FILENAME_LENGTH]; FILE *file; int line = 1; ! char *x, *msg = strdup(message); ! if (msg == NULL) { ! LOG(LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } ! if (quarantine_lock (username)) { ! free(msg); return EFILE; } --- 2151,2167 ---- char filename[MAX_FILENAME_LENGTH]; FILE *file; int line = 1; ! char *x, *msg = strdup (message); ! if (msg == NULL) ! { ! LOG (LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } ! if (quarantine_lock (username)) ! { ! free (msg); return EFILE; } *************** *** 2075,2081 **** { file_error (ERROR_FILE_WRITE, filename, strerror (errno)); quarantine_unlock (username); ! free(msg); return EFILE; } --- 2171,2177 ---- { file_error (ERROR_FILE_WRITE, filename, strerror (errno)); quarantine_unlock (username); ! free (msg); return EFILE; } *************** *** 2089,2109 **** fputs (head, file); } ! x = strsep(&msg, "\n"); ! while(x != NULL) { ! if (!strncmp(x, "From ", 5) && line != 1) ! fputs(">", file); ! fputs(x, file); ! fputs("\n", file); line++; ! x = strsep(&msg, "\n"); } fputs ("\n\n", file); fclose (file); quarantine_unlock (username); ! free(msg); return 0; } --- 2185,2206 ---- fputs (head, file); } ! x = strsep (&msg, "\n"); ! while (x != NULL) ! { ! if (!strncmp (x, "From ", 5) && line != 1) ! fputs (">", file); ! fputs (x, file); ! fputs ("\n", file); line++; ! x = strsep (&msg, "\n"); } fputs ("\n\n", file); fclose (file); quarantine_unlock (username); ! free (msg); return 0; } *************** *** 2128,2140 **** return EFILE; } ! fprintf (file, "%ld,%ld,%ld,%ld,%ld,%ld\n", ! totals->total_spam - (totals->spam_misclassified + totals->spam_corpusfed), ! totals->total_innocent - (totals->innocent_misclassified + totals->innocent_corpusfed), ! totals->spam_misclassified, ! totals->innocent_misclassified, ! totals->spam_corpusfed, ! totals->innocent_corpusfed); fclose (file); return 0; } --- 2225,2237 ---- return EFILE; } ! fprintf (file, "%ld,%ld,%ld,%ld,%ld,%ld\n", ! totals->total_spam - (totals->spam_misclassified + ! totals->spam_corpusfed), ! totals->total_innocent - (totals->innocent_misclassified + ! totals->innocent_corpusfed), ! totals->spam_misclassified, totals->innocent_misclassified, ! totals->spam_corpusfed, totals->innocent_corpusfed); fclose (file); return 0; } *************** *** 2282,2345 **** parameters: only SIG _OR_ message should be passed. the other should be left NULL depending on whether we have a signature or a message ! */ int ! inoculate_user (const char *username, struct _ds_spam_signature *SIG, const char *message) { DSPAM_CTX *INOC; int do_inoc = 1, result = 0; ! LOGDEBUG("checking to see if user %s requires this inoculation", username); ! /* First see if the user needs to be inoculated */ #ifdef CHAINED_TOKENS INOC = dspam_init (username, NULL, DSM_CLASSIFY, DSF_CHAINED); #else INOC = dspam_init (username, NULL, DSM_CLASSIFY, 0); #endif ! if (INOC) { ! if (SIG) { INOC->flags = INOC->flags ^ DSF_SIGNATURE; INOC->signature = SIG; result = dspam_process (INOC, NULL); ! } else { INOC->flags = INOC->flags ^ DSF_CORPUS; result = dspam_process (INOC, message); } ! if (!result && INOC->result == DSR_ISSPAM) do_inoc = 0; if (SIG) INOC->signature = NULL; ! dspam_destroy(INOC); } ! ! if (!do_inoc) { ! LOGDEBUG("skipping user %s: doesn't require inoculation", username); return EFAILURE; ! } else { ! LOGDEBUG("inoculating user %s", username); #ifdef CHAINED_TOKENS INOC = dspam_init (username, NULL, DSM_ADDSPAM, DSF_CHAINED | DSF_INOCULATE); #else INOC = dspam_init (username, NULL, DSM_ADDSPAM, DSF_INOCULATE); #endif ! if (INOC) { ! if (SIG) { INOC->flags = INOC->flags ^ DSF_SIGNATURE; INOC->signature = SIG; result = dspam_process (INOC, NULL); ! } else { INOC->flags = INOC->flags ^ DSF_CORPUS; result = dspam_process (INOC, message); } if (SIG) INOC->signature = NULL; ! dspam_destroy(INOC); } } --- 2379,2454 ---- parameters: only SIG _OR_ message should be passed. the other should be left NULL depending on whether we have a signature or a message ! */ int ! inoculate_user (const char *username, struct _ds_spam_signature *SIG, ! const char *message) { DSPAM_CTX *INOC; int do_inoc = 1, result = 0; ! LOGDEBUG ("checking to see if user %s requires this inoculation", username); ! /* First see if the user needs to be inoculated */ #ifdef CHAINED_TOKENS INOC = dspam_init (username, NULL, DSM_CLASSIFY, DSF_CHAINED); #else INOC = dspam_init (username, NULL, DSM_CLASSIFY, 0); #endif ! if (INOC) ! { ! if (SIG) ! { INOC->flags = INOC->flags ^ DSF_SIGNATURE; INOC->signature = SIG; result = dspam_process (INOC, NULL); ! } ! else ! { INOC->flags = INOC->flags ^ DSF_CORPUS; result = dspam_process (INOC, message); } ! if (!result && INOC->result == DSR_ISSPAM) do_inoc = 0; if (SIG) INOC->signature = NULL; ! dspam_destroy (INOC); } ! ! if (!do_inoc) ! { ! LOGDEBUG ("skipping user %s: doesn't require inoculation", username); return EFAILURE; ! } ! else ! { ! LOGDEBUG ("inoculating user %s", username); #ifdef CHAINED_TOKENS INOC = dspam_init (username, NULL, DSM_ADDSPAM, DSF_CHAINED | DSF_INOCULATE); #else INOC = dspam_init (username, NULL, DSM_ADDSPAM, DSF_INOCULATE); #endif ! if (INOC) ! { ! if (SIG) ! { INOC->flags = INOC->flags ^ DSF_SIGNATURE; INOC->signature = SIG; result = dspam_process (INOC, NULL); ! } ! else ! { INOC->flags = INOC->flags ^ DSF_CORPUS; result = dspam_process (INOC, message); } if (SIG) INOC->signature = NULL; ! dspam_destroy (INOC); } } *************** *** 2354,2376 **** int ! user_classify (const char *username, struct _ds_spam_signature *SIG, const char *message) { DSPAM_CTX *CLX; int result = 0; ! /* First see if the user needs to be inoculated */ #ifdef CHAINED_TOKENS CLX = dspam_init (username, NULL, DSM_CLASSIFY, DSF_CHAINED); #else CLX = dspam_init (username, NULL, DSM_CLASSIFY, 0); #endif ! if (CLX) { ! if (SIG) { CLX->flags = CLX->flags ^ DSF_SIGNATURE; CLX->signature = SIG; result = dspam_process (CLX, NULL); ! } else { CLX->flags = CLX->flags ^ DSF_CORPUS; result = dspam_process (CLX, message); } --- 2463,2490 ---- int ! user_classify (const char *username, struct _ds_spam_signature *SIG, ! const char *message) { DSPAM_CTX *CLX; int result = 0; ! /* First see if the user needs to be inoculated */ #ifdef CHAINED_TOKENS CLX = dspam_init (username, NULL, DSM_CLASSIFY, DSF_CHAINED); #else CLX = dspam_init (username, NULL, DSM_CLASSIFY, 0); #endif ! if (CLX) ! { ! if (SIG) ! { CLX->flags = CLX->flags ^ DSF_SIGNATURE; CLX->signature = SIG; result = dspam_process (CLX, NULL); ! } ! else ! { CLX->flags = CLX->flags ^ DSF_CORPUS; result = dspam_process (CLX, message); } *************** *** 2378,2391 **** if (SIG) CLX->signature = NULL; ! if (result) { ! LOGDEBUG("user_classify() returned error %d", result); result = 0; } else result = CLX->result; ! ! dspam_destroy(CLX); } return result; --- 2492,2506 ---- if (SIG) CLX->signature = NULL; ! if (result) ! { ! LOGDEBUG ("user_classify() returned error %d", result); result = 0; } else result = CLX->result; ! ! dspam_destroy (CLX); } return result; Index: dspam/dspam.h diff -c dspam/dspam.h:1.1.1.4 dspam/dspam.h:1.1.1.4.2.1 *** dspam/dspam.h:1.1.1.4 Fri Oct 31 15:34:14 2003 --- dspam/dspam.h Sat Nov 15 18:34:58 2003 *************** *** 53,66 **** /* Public agent functions */ int deliver_message (const char *message, const char *mailer_args, const char *username, int mode); ! int process_message (char *group, size_t size, buffer * message, const char *username, int mode, int flags); int write_web_stats (const char *username, struct _ds_spam_totals *totals); int quarantine_message (const char *message, const char *username); int quarantine_unlock (const char *username); int quarantine_lock (const char *username); ! int inoculate_user (const char *username, struct _ds_spam_signature *SIG, const char *message); ! int user_classify (const char *username, struct _ds_spam_signature *SIG, const char *message); /* DO NOT MODIFY BELOW THIS LINE */ --- 53,68 ---- /* Public agent functions */ int deliver_message (const char *message, const char *mailer_args, const char *username, int mode); ! int process_message (char *group, size_t size, buffer * message, const char *username, int mode, int flags); int write_web_stats (const char *username, struct _ds_spam_totals *totals); int quarantine_message (const char *message, const char *username); int quarantine_unlock (const char *username); int quarantine_lock (const char *username); ! int inoculate_user (const char *username, struct _ds_spam_signature *SIG, ! const char *message); ! int user_classify (const char *username, struct _ds_spam_signature *SIG, ! const char *message); /* DO NOT MODIFY BELOW THIS LINE */ Index: dspam/dspam.spec diff -c /dev/null dspam/dspam.spec:1.49.4.2 *** /dev/null Sat Nov 15 20:45:12 2003 --- dspam/dspam.spec Sat Nov 15 20:40:53 2003 *************** *** 0 **** --- 1,292 ---- + %ifos Linux + %define sendmailcf /usr/share/sendmail-cf + %define cgibin /var/www/cgi-bin + %define htmldir /var/www/html + %else + %define sendmailcf /usr/lib/sendmail-cf + %define cgibin /usr/local/www/cgi-bin + %define htmldir /Public + %endif + + Summary: A library and Mail Delivery Agent for Bayesian spam filtering + Name: dspam + Version: 2.8.beta.2 + Release: 1 + Copyright: GPL + URL: http://www.networkdweebs.com/software/dspam/ + Group: System Environment/Daemons + Source: http://bmsi.com/linux/dspam-%{version}.tar.gz + Source1: dspam.m4 + Patch: dspam-2.8.patch + Buildroot: /var/tmp/dspam-root + %ifos Linux + BuildRequires: db3-devel patch + Requires: /usr/sbin/useradd + %else + %ifos aix4.1 + BuildRequires: db3-devel patch + %else + BuildRequires: db4-devel patch + %endif + %endif + + %package devel + Summary: Developers library for custom access to dspam + Group: Development/Libraries + + %description + DSPAM (as in De-Spam) is an open-source project to create a new kind of + anti-spam mechanism, and is currently effective as both a server-side agent + for UNIX email servers and a developer's library for mail clients, other + anti-spam tools, and similar projects requiring drop-in spam filtering. + + The DSPAM agent masquerades as the email server's local delivery agent and + filters/learns spams using an advanced Bayesian statistical approach (based on + Baye's theorem of combined probabilities) which provides an administratively + maintenance-free, easy-learning Anti-Spam service custom tailored to each + individual user's behavior. Advanced because on top of standard Bayesian + filtering is also incorporated the use of Chained Tokens, de-obfuscation, and + other enhancements. DSPAM works great with Sendmail and Exim, and should work + well with any other MTA that supports an external local delivery agent + (postfix, qmail, etc.) + + %description devel + DSPAM has had its core engine moved into a separate library, libdspam. + This library can be used by developers to provide 'drop-in' spam filtering for + their mail client applications, other anti-spam tools, or similar projects. + + %prep + %setup -q + %patch -p0 + #%patch1 -p1 + + %build + %ifos aix4.1 + export CC="gcc -mthreads" + LDFLAGS="-Wl,-blibpath:/lib:/usr/local/lib" + %else + LDFLAGS=-s + %endif + CFLAGS="$RPM_OPT_FLAGS" + export CFLAGS LDFLAGS + ./configure --with-userdir=/var/lib/dspam \ + --with-userdir-owner=none \ + --with-userdir-group=none \ + --with-dspam-owner=none \ + --with-dspam-group=none \ + %ifos aix4.1 + --with-local-delivery-agent=/bin/bellmail \ + %endif + --with-storage-driver=libdb3_drv \ + --disable-dependency-tracking + + make + mv dspam dspam.optout + rm dspam.o + make dspam CPPFLAGS=-DOPT_IN + ln dspam dspam.optin + + %install + rm -rf $RPM_BUILD_ROOT + make install DESTDIR=$RPM_BUILD_ROOT + + # include both optin and optout version of dspam + cp dspam.optout $RPM_BUILD_ROOT/usr/local/bin + cd $RPM_BUILD_ROOT/usr/local/bin + mv dspam dspam.optin + ln -s dspam.optout dspam + cd - + + # allow others to query stats + chmod g+s $RPM_BUILD_ROOT/usr/local/bin/dspam_stats + + # manually copy include files needed for devel package + INCDIR="$RPM_BUILD_ROOT/usr/local/include" + mkdir -p $INCDIR + cp -p libdspam.h libdspam_objects.h lht.h nodetree.h $INCDIR + + # provide maintenance scripts + ETCDIR="$RPM_BUILD_ROOT/etc" + mkdir -p $ETCDIR/cron.hourly + mkdir -p $ETCDIR/cron.daily + mkdir -p $ETCDIR/cron.weekly + cat >$ETCDIR/cron.daily/dspam <<'EOF' + #!/bin/sh + /usr/local/bin/dspam_clean + EOF + chmod a+x $ETCDIR/cron.daily/dspam + cat >$ETCDIR/cron.weekly/dspam <<'EOF' + #!/bin/sh + /usr/local/bin/dspam_purge + EOF + chmod a+x $ETCDIR/cron.weekly/dspam + cat >$ETCDIR/cron.hourly/dspam <<'EOF' + #!/bin/sh + cd /var/lib/dspam + exec >>reprocess.log 2>&1 + /usr/local/bin/pydspam_process *.spam *.fp + EOF + chmod a+x $ETCDIR/cron.hourly/dspam + + # install script for optional smart spam alias + cp -p addspam.sh $RPM_BUILD_ROOT/usr/local/bin/addspam + cd $RPM_BUILD_ROOT/usr/local/bin + ln addspam falsepositive + cd - + mkdir -p $RPM_BUILD_ROOT/var/log + touch $RPM_BUILD_ROOT/var/log/dspam.log + + # allow dspam in /etc/smrsh + mkdir -p $ETCDIR/smrsh + ln -sf /usr/local/bin/dspam $ETCDIR/smrsh + ln -sf /usr/local/bin/addspam $ETCDIR/smrsh + + # install sendmail mailer + mkdir -p $RPM_BUILD_ROOT%{sendmailcf}/mailer + cp -p %{SOURCE1} $RPM_BUILD_ROOT%{sendmailcf}/mailer + + # install CGI script + CGIDIR="$RPM_BUILD_ROOT%{cgibin}" + HTMLDIR="$RPM_BUILD_ROOT%{htmldir}" + mkdir -p $HTMLDIR/dspam + mkdir -p $CGIDIR + mkdir -p $RPM_BUILD_ROOT/etc/mail + ln -sf /var/lib/dspam $RPM_BUILD_ROOT/etc/mail/dspam + cp -p cgi/* $HTMLDIR/dspam + %ifos aix4.1 + # No suexec on our AIX installs + cat >$CGIDIR/dspam.cgi <<'EOF' + #!/bin/sh + cd %{htmldir}/dspam + exec /usr/local/bin/perl dspam.cgi + EOF + %else + # Use suexec to run CGI + cat >$CGIDIR/dspam.cgi <<'EOF' + #!/bin/sh + cd %{htmldir}/dspam + exec /usr/sbin/suexec dspam dspam dspam.cgi + EOF + %endif + chmod 0755 $HTMLDIR/dspam $HTMLDIR/dspam/dspam.cgi + + %clean + rm -rf $RPM_BUILD_ROOT + + %ifos linux + %pre + /usr/sbin/useradd -G mail -d /var/lib/dspam -c "Dspam agent" -s /dev/null \ + dspam >/dev/null 2>&1 || : + + %post + if grep '^/usr/local/lib$' /etc/ld.so.conf >/dev/null; then + : + else + echo "/usr/local/lib" >>/etc/ld.so.conf + fi + /sbin/ldconfig + %endif + %ifos aix4.1 + %pre + mkuser -a pgrp=mail home=/var/lib/dspam \ + gecos="DSpam mail filter" dspam 2>/dev/null || : + %endif + + %files + %defattr(-,root,root) + %doc README CHANGE CHANGE.sdg dspam-button.gif + %ifnos aix4.1 + /usr/local/lib/libdspam.so.3.0.0 + /usr/local/lib/libdspam.so.3 + %endif + %attr(02511,root,mail)/usr/local/bin/dspam.optin + %attr(02511,root,mail)/usr/local/bin/dspam.optout + %attr(-,root,mail)/usr/local/bin/dspam + %attr(-,root,mail)/usr/local/bin/dspam_clean + %attr(-,root,mail)/usr/local/bin/dspam_convert + %attr(-,root,mail)/usr/local/bin/dspam_crc + %attr(-,root,mail)/usr/local/bin/dspam_dump + %attr(-,root,mail)/usr/local/bin/dspam_purge + %attr(-,root,mail)/usr/local/bin/dspam_stats + /usr/local/bin/dspam_corpus + /usr/local/bin/dspam_genaliases + %attr(0775,root,mail) /var/lib/dspam + /etc/cron.daily/dspam + /etc/cron.weekly/dspam + /etc/smrsh/dspam + /etc/smrsh/addspam + %{sendmailcf}/mailer/* + %attr(-,dspam,dspam)%{htmldir}/dspam + %attr(0755,root,root)%{cgibin}/dspam.cgi + /etc/mail/dspam + %config %attr(0755,root,mail)/usr/local/bin/addspam + %config %attr(0755,root,mail)/usr/local/bin/falsepositive + %attr(0664,root,mail)/var/log/addspam + + %files devel + %defattr(-,root,root) + %ifnos aix4.1 + /usr/local/lib/libdspam.so + %endif + /usr/local/lib/libdspam.la + /usr/local/lib/libdspam.a + /usr/local/include/* + + %changelog + * Sat Nov 15 2003 Stuart Gathman 2.8.beta.2-1 + - remove python subpackage, moved to pydspam RPM + - Support 2.8 + * Tue Oct 21 2003 Stuart Gathman 2.6.5.2-4 + - pydspam-1.1.4 + - run pydspam_process on the hour + - Count signature spam corpus as miss + - Remove "Delete All" from CGI and default messages to checked. + * Wed Sep 10 2003 Stuart Gathman + - Fix memory leaks + - Increase lock timeout + - Make dspam sgid and a+x so that generic addspam works + - Install optin and optout versions. + * Sat Sep 06 2003 Stuart Gathman + - Merge dspam-2.6.5.2 + - Move cgi to /var/www/html/dspam. logo and css weren't getting + - found under cgi-bin. + * Fri Sep 05 2003 Stuart Gathman + - Modify tbt.c to use parent pointer and eliminate recursion which + - was overflowing thread stack on AIX + * Tue Sep 02 2003 Stuart Gathman + - Merge changes for release 2.6.5 + - use pydspam 1.1.1 + * Wed Aug 27 2003 Stuart Gathman + - Tweak for AIX + * Thu Aug 18 2003 Stuart Gathman + - Merge changes for 2.6.4.01 + - empty input patch + - Include smart spam alias + * Thu Aug 14 2003 Stuart Gathman + - Merge changes for 2.6.4 + * Mon Aug 04 2003 Stuart Gathman + - Install CGI script to run as dspam user + * Thu Jul 31 2003 Stuart Gathman + - Make building python package optional + - OK, OK, so maybe it should be a separate RPM + * Wed Jul 30 2003 Stuart Gathman + - Fix dspam_stats bug for release 2 + * Wed Jul 30 2003 Stuart Gathman + - Move python source to pydspam project + - merge dspam-2.6.2.02 from networkdweebs + * Fri Jul 11 2003 Stuart Gathman + - Move python support to sub package + - fix CORPUS bug + * Thu Jul 10 2003 Stuart Gathman + - Bug fixes, python support. + * Thu Jul 03 2003 Stuart Gathman + - Merge with 2.6.2 stable + * Wed Jul 02 2003 Stuart Gathman + - Fix bugs in DSF_CLASSIFY + * Mon Jun 30 2003 Stuart Gathman + - Fix bugs in dspam.c and libdspam.c + * Thu Jun 26 2003 Stuart Gathman + - Add dspam to /etc/smrsh + - Add dspam mailer to sendmail-cf + * Wed Jun 25 2003 Stuart Gathman + - Linux RPM Index: dspam/error.c diff -c dspam/error.c:1.1.1.4 dspam/error.c:1.1.1.4.2.1 *** dspam/error.c:1.1.1.4 Fri Oct 17 16:08:03 2003 --- dspam/error.c Sat Nov 15 18:34:58 2003 *************** *** 98,104 **** char ct[128]; char fn[MAX_FILENAME_LENGTH]; ! snprintf(fn, sizeof(fn), "%s/dspam.debug", USERDIR); tm = time (NULL); snprintf (ct, sizeof (ct), "%s", ctime (&tm)); --- 98,104 ---- char ct[128]; char fn[MAX_FILENAME_LENGTH]; ! snprintf (fn, sizeof (fn), "%s/dspam.debug", USERDIR); tm = time (NULL); snprintf (ct, sizeof (ct), "%s", ctime (&tm)); Index: dspam/example.c diff -c dspam/example.c:1.1.1.4 dspam/example.c:1.1.1.4.2.1 *** dspam/example.c:1.1.1.4 Fri Oct 24 14:53:10 2003 --- dspam/example.c Sat Nov 15 18:34:58 2003 *************** *** 20,27 **** struct _ds_spam_signature SIG; /* Example signature */ /* Performs any driver-specific startup functions ! (such as initializing internal lock tables) */ ! dspam_init_driver(); /* read in the message from stdin */ message[0] = 0; --- 20,27 ---- struct _ds_spam_signature SIG; /* Example signature */ /* Performs any driver-specific startup functions ! * (such as initializing internal lock tables) */ ! dspam_init_driver (); /* read in the message from stdin */ message[0] = 0; *************** *** 210,216 **** } /* Performs any driver-specific shutdown functions */ ! dspam_shutdown_driver(); exit (EXIT_SUCCESS); --- 210,216 ---- } /* Performs any driver-specific shutdown functions */ ! dspam_shutdown_driver (); exit (EXIT_SUCCESS); Index: dspam/libdb3_drv.c diff -c dspam/libdb3_drv.c:1.1.1.1 dspam/libdb3_drv.c:1.1.1.1.2.1 *** dspam/libdb3_drv.c:1.1.1.1 Mon Nov 3 09:57:27 2003 --- dspam/libdb3_drv.c Sat Nov 15 18:34:58 2003 *************** *** 59,66 **** int dspam_init_driver (void) { ! _libdb3_drv_locks = nt_create(NT_PTR); ! if (_libdb3_drv_locks == NULL) return EUNKNOWN; return 0; --- 59,66 ---- int dspam_init_driver (void) { ! _libdb3_drv_locks = nt_create (NT_PTR); ! if (_libdb3_drv_locks == NULL) return EUNKNOWN; return 0; *************** *** 69,75 **** int dspam_shutdown_driver (void) { ! nt_destroy(_libdb3_drv_locks); return 0; } --- 69,75 ---- int dspam_shutdown_driver (void) { ! nt_destroy (_libdb3_drv_locks); return 0; } *************** *** 94,112 **** ret = (*s->db->get) (s->db, NULL, &key, &data, 0); if (ret) { ! if (ret == DB_RUNRECOVERY) { ! if (! _libdb3_drv_recover(CTX, 1)) { return _libdb3_drv_get_spamtotals (CTX); ! } else { return EUNKNOWN; } ! } else { LOGDEBUG ("_ds_get_totals: db->get failed: %s", db_strerror (ret)); return EFAILURE; } } ! memset(&CTX->totals, 0, sizeof(struct _ds_spam_totals)); memcpy (&CTX->totals, data.data, data.size); return 0; --- 94,118 ---- ret = (*s->db->get) (s->db, NULL, &key, &data, 0); if (ret) { ! if (ret == DB_RUNRECOVERY) ! { ! if (!_libdb3_drv_recover (CTX, 1)) ! { return _libdb3_drv_get_spamtotals (CTX); ! } ! else ! { return EUNKNOWN; } ! } ! else ! { LOGDEBUG ("_ds_get_totals: db->get failed: %s", db_strerror (ret)); return EFAILURE; } } ! memset (&CTX->totals, 0, sizeof (struct _ds_spam_totals)); memcpy (&CTX->totals, data.data, data.size); return 0; *************** *** 141,153 **** ret = (*s->db->put) (s->db, NULL, &key, &data, 0); if (ret) { ! if (ret == DB_RUNRECOVERY) { ! if (! _libdb3_drv_recover(CTX, 1)) { return _libdb3_drv_set_spamtotals (CTX); ! } else { return EUNKNOWN; } ! } else { LOGDEBUG ("_ds_set_totals: db->set failed: %s", db_strerror (ret)); return EFAILURE; } --- 147,165 ---- ret = (*s->db->put) (s->db, NULL, &key, &data, 0); if (ret) { ! if (ret == DB_RUNRECOVERY) ! { ! if (!_libdb3_drv_recover (CTX, 1)) ! { return _libdb3_drv_set_spamtotals (CTX); ! } ! else ! { return EUNKNOWN; } ! } ! else ! { LOGDEBUG ("_ds_set_totals: db->set failed: %s", db_strerror (ret)); return EFAILURE; } *************** *** 234,247 **** ret = (*s->db->get) (s->db, NULL, &key, &data, 0); ! if (ret) { ! if (ret == DB_RUNRECOVERY) { ! if (! _libdb3_drv_recover(CTX, 1)) { return _ds_get_spamrecord (CTX, token, stat); ! } else { return EUNKNOWN; } ! } else { return EFAILURE; } } --- 246,266 ---- ret = (*s->db->get) (s->db, NULL, &key, &data, 0); ! if (ret) ! { ! if (ret == DB_RUNRECOVERY) ! { ! if (!_libdb3_drv_recover (CTX, 1)) ! { return _ds_get_spamrecord (CTX, token, stat); ! } ! else ! { return EUNKNOWN; } ! } ! else ! { return EFAILURE; } } *************** *** 298,310 **** if (ret) { ! if (ret == DB_RUNRECOVERY) { ! if (! _libdb3_drv_recover(CTX, 1)) { return _ds_set_spamrecord (CTX, token, stat); ! } else { return EUNKNOWN; } ! } else { LOGDEBUG ("_ds_set_spamrecord: db->put failed: %s", db_strerror (ret)); return EFAILURE; } --- 317,335 ---- if (ret) { ! if (ret == DB_RUNRECOVERY) ! { ! if (!_libdb3_drv_recover (CTX, 1)) ! { return _ds_set_spamrecord (CTX, token, stat); ! } ! else ! { return EUNKNOWN; } ! } ! else ! { LOGDEBUG ("_ds_set_spamrecord: db->put failed: %s", db_strerror (ret)); return EFAILURE; } *************** *** 343,350 **** } else { ! LOGDEBUG("entering recovery mode"); ! free(CTX->storage); s = malloc (sizeof (struct _libdb3_drv_storage)); if (s == NULL) { --- 368,375 ---- } else { ! LOGDEBUG ("entering recovery mode"); ! free (CTX->storage); s = malloc (sizeof (struct _libdb3_drv_storage)); if (s == NULL) { *************** *** 352,358 **** LOG (LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } ! s->db = NULL; s->sig = NULL; s->env = NULL; --- 377,383 ---- LOG (LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } ! s->db = NULL; s->sig = NULL; s->env = NULL; *************** *** 395,426 **** } */ ! if (!recovery) { CTX->result = ! s->env->open (s->env, db_home, DB_CREATE | DB_INIT_LOCK | ! DB_INIT_MPOOL, 0); ! } else if (recovery == 1) { CTX->result = s->env->open (s->env, db_home, ! DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER_FATAL, 0); ! } else { CTX->result = s->env->open (s->env, db_home, ! DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER, 0); } ! if (CTX->result) { if (recovery || CTX->result != DB_RUNRECOVERY) { ! LOG (LOG_WARNING, "DB_ENV->open failed: %s", db_strerror (CTX->result)); return EFAILURE; } else { ! recovery = 1; ! return _ds_init_storage (CTX); } } --- 420,458 ---- } */ ! if (!recovery) ! { CTX->result = ! s->env->open (s->env, db_home, DB_CREATE | DB_INIT_LOCK | ! DB_INIT_MPOOL, 0); ! } ! else if (recovery == 1) ! { CTX->result = s->env->open (s->env, db_home, ! DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER_FATAL, 0); ! } ! else ! { CTX->result = s->env->open (s->env, db_home, ! DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER, 0); } ! if (CTX->result) ! { if (recovery || CTX->result != DB_RUNRECOVERY) { ! LOG (LOG_WARNING, "DB_ENV->open failed: %s", ! db_strerror (CTX->result)); return EFAILURE; } else { ! recovery = 1; ! return _ds_init_storage (CTX); } } *************** *** 445,451 **** else { recovery = 1; ! s->env->close(s->env, 0); return _ds_init_storage (CTX); } --- 477,483 ---- else { recovery = 1; ! s->env->close (s->env, 0); return _ds_init_storage (CTX); } *************** *** 474,530 **** else { recovery = 1; ! s->db->close(s->db, 0); ! s->env->close(s->env, 0); return _ds_init_storage (CTX); } } /* Obtain a lock for this user */ ! lock_result = _libdb3_drv_lock_get(CTX->username); ! if (lock_result < 0) { ! LOGDEBUG("locking subsystem returned error"); return EUNKNOWN; } ! ! if (lock_result > 0) { int i = 0; ! LOGDEBUG("acquiring lock"); /* Run two instances of lock detection */ ! lock_detect(s->env, 0, DB_LOCK_OLDEST, NULL); ! lock_detect(s->env, 0, DB_LOCK_OLDEST, NULL); s->locker = 0; if ((CTX->result = lock_id (s->env, &s->locker)) != 0) { LOG (LOG_WARNING, "unable to get a locker id for %s: %s", ! CTX->username, db_strerror (CTX->result)); return EUNKNOWN; } dbt.data = CTX->username; dbt.size = strlen (CTX->username); while ((CTX->result = ! lock_get (s->env, s->locker, DB_LOCK_NOWAIT, &dbt, ! DB_LOCK_WRITE, &s->lock)) != 0 && i<120) { ! LOGDEBUG("spinning lock %d/120", i); i++; ! lock_detect(s->env, 0, DB_LOCK_OLDEST, NULL); ! lock_detect(s->env, 0, DB_LOCK_OLDEST, NULL); ! sleep(1); } ! if (CTX->result) { ! if (!recovery) { recovery = 2; ! return _ds_init_storage(CTX); ! } else { LOG (LOG_WARNING, ! "unable to obtain database lock for user %s: %s", CTX->username, db_strerror (CTX->result)); return EUNKNOWN; } --- 506,568 ---- else { recovery = 1; ! s->db->close (s->db, 0); ! s->env->close (s->env, 0); return _ds_init_storage (CTX); } } /* Obtain a lock for this user */ ! lock_result = _libdb3_drv_lock_get (CTX->username); ! if (lock_result < 0) ! { ! LOGDEBUG ("locking subsystem returned error"); return EUNKNOWN; } ! ! if (lock_result > 0) ! { int i = 0; ! LOGDEBUG ("acquiring lock"); /* Run two instances of lock detection */ ! lock_detect (s->env, 0, DB_LOCK_OLDEST, NULL); ! lock_detect (s->env, 0, DB_LOCK_OLDEST, NULL); s->locker = 0; if ((CTX->result = lock_id (s->env, &s->locker)) != 0) { LOG (LOG_WARNING, "unable to get a locker id for %s: %s", ! CTX->username, db_strerror (CTX->result)); return EUNKNOWN; } dbt.data = CTX->username; dbt.size = strlen (CTX->username); while ((CTX->result = ! lock_get (s->env, s->locker, DB_LOCK_NOWAIT, &dbt, ! DB_LOCK_WRITE, &s->lock)) != 0 && i < 120) { ! LOGDEBUG ("spinning lock %d/120", i); i++; ! lock_detect (s->env, 0, DB_LOCK_OLDEST, NULL); ! lock_detect (s->env, 0, DB_LOCK_OLDEST, NULL); ! sleep (1); } ! if (CTX->result) ! { ! if (!recovery) ! { recovery = 2; ! return _ds_init_storage (CTX); ! } ! else ! { LOG (LOG_WARNING, ! "unable to obtain database lock for user %s: %s", CTX->username, db_strerror (CTX->result)); return EUNKNOWN; } *************** *** 555,564 **** memset (&CTX->totals, 0, sizeof (struct _ds_spam_totals)); } ! if (recovery) { recovery = 0; ! _ds_shutdown_storage(CTX); ! return _ds_init_storage(CTX); } return 0; --- 593,603 ---- memset (&CTX->totals, 0, sizeof (struct _ds_spam_totals)); } ! if (recovery) ! { recovery = 0; ! _ds_shutdown_storage (CTX); ! return _ds_init_storage (CTX); } return 0; *************** *** 599,612 **** { int lock_result; ! lock_result = _libdb3_drv_lock_put(CTX->username); ! if (lock_result < 0) { ! LOGDEBUG("locking subsystem returned error"); return EUNKNOWN; } ! if (lock_result > 0) { ! LOGDEBUG("freeing lock"); lock_put (s->env, &s->lock); } ret = s->db->close (s->db, 0); --- 638,653 ---- { int lock_result; ! lock_result = _libdb3_drv_lock_put (CTX->username); ! if (lock_result < 0) ! { ! LOGDEBUG ("locking subsystem returned error"); return EUNKNOWN; } ! if (lock_result > 0) ! { ! LOGDEBUG ("freeing lock"); lock_put (s->env, &s->lock); } ret = s->db->close (s->db, 0); *************** *** 623,629 **** ret = s->env->close (s->env, 0); } ! if (!ret) { free (CTX->storage); CTX->storage = NULL; --- 664,670 ---- ret = s->env->close (s->env, 0); } ! if (!ret) { free (CTX->storage); CTX->storage = NULL; *************** *** 1044,1074 **** */ int ! _libdb3_drv_lock_get (const char *username) { struct nt_node *node; struct nt_c c; struct _libdb3_drv_lock *lock; ! node = c_nt_first(_libdb3_drv_locks, &c); ! while(node != NULL) { lock = (struct _libdb3_drv_lock *) node->ptr; ! if (lock != NULL && !strcmp(username, lock->username)) { lock->readers++; ! if (lock->readers==1) return 1; return 0; } ! node = c_nt_next(_libdb3_drv_locks, &c); } ! lock = malloc(sizeof(struct _libdb3_drv_lock)); ! if (lock == NULL) return EUNKNOWN; ! strcpy(lock->username, username); lock->readers = 1; ! nt_add(_libdb3_drv_locks, (void *) lock); return 1; } --- 1085,1117 ---- */ int ! _libdb3_drv_lock_get (const char *username) { struct nt_node *node; struct nt_c c; struct _libdb3_drv_lock *lock; ! node = c_nt_first (_libdb3_drv_locks, &c); ! while (node != NULL) ! { lock = (struct _libdb3_drv_lock *) node->ptr; ! if (lock != NULL && !strcmp (username, lock->username)) ! { lock->readers++; ! if (lock->readers == 1) return 1; return 0; } ! node = c_nt_next (_libdb3_drv_locks, &c); } ! lock = malloc (sizeof (struct _libdb3_drv_lock)); ! if (lock == NULL) return EUNKNOWN; ! strcpy (lock->username, username); lock->readers = 1; ! nt_add (_libdb3_drv_locks, (void *) lock); return 1; } *************** *** 1076,1098 **** 1: Free lock */ int ! _libdb3_drv_lock_put (const char *username) { struct nt_node *node; struct nt_c c; struct _libdb3_drv_lock *lock; ! ! node = c_nt_first(_libdb3_drv_locks, &c); ! while(node != NULL) { lock = (struct _libdb3_drv_lock *) node->ptr; ! if (lock != NULL && !strcmp(username, lock->username)) { lock->readers--; if (lock->readers == 0) return 1; return 0; } ! node = c_nt_next(_libdb3_drv_locks, &c); } return EFAILURE; --- 1119,1143 ---- 1: Free lock */ int ! _libdb3_drv_lock_put (const char *username) { struct nt_node *node; struct nt_c c; struct _libdb3_drv_lock *lock; ! ! node = c_nt_first (_libdb3_drv_locks, &c); ! while (node != NULL) ! { lock = (struct _libdb3_drv_lock *) node->ptr; ! if (lock != NULL && !strcmp (username, lock->username)) ! { lock->readers--; if (lock->readers == 0) return 1; return 0; } ! node = c_nt_next (_libdb3_drv_locks, &c); } return EFAILURE; *************** *** 1106,1114 **** int dbflag = DB_CREATE; int lock_result; ! _ds_shutdown_storage(CTX); ! LOGDEBUG("recovering database"); s = malloc (sizeof (struct _libdb3_drv_storage)); if (s == NULL) { --- 1151,1159 ---- int dbflag = DB_CREATE; int lock_result; ! _ds_shutdown_storage (CTX); ! LOGDEBUG ("recovering database"); s = malloc (sizeof (struct _libdb3_drv_storage)); if (s == NULL) { *************** *** 1116,1122 **** LOG (LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } ! s->db = NULL; s->sig = NULL; s->env = NULL; --- 1161,1167 ---- LOG (LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } ! s->db = NULL; s->sig = NULL; s->env = NULL; *************** *** 1124,1130 **** if (CTX->username != NULL) { char db_home[MAX_FILENAME_LENGTH]; ! if (CTX->group == NULL) { strcpy (s->dictionary, _ds_userdir_path (CTX->username, "dict")); --- 1169,1175 ---- if (CTX->username != NULL) { char db_home[MAX_FILENAME_LENGTH]; ! if (CTX->group == NULL) { strcpy (s->dictionary, _ds_userdir_path (CTX->username, "dict")); *************** *** 1147,1184 **** return EFAILURE; } ! if (fatal) { CTX->result = s->env->open (s->env, db_home, ! DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER_FATAL, 0); ! } else { CTX->result = s->env->open (s->env, db_home, DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER, 0); } ! if (CTX->result) { ! LOG (LOG_WARNING, "DB_ENV->open failed: %s", db_strerror (CTX->result)); ! return EFAILURE; } if ((CTX->result = db_create (&s->db, s->env, 0)) != 0) { LOG (LOG_WARNING, "db_create failed: %s", db_strerror (CTX->result)); s->db = NULL; ! return EFAILURE; } if ((CTX->result = s->db->DBOPEN (s->db, NULL, s->dictionary, NULL, DB_BTREE, dbflag, 0)) != 0) ! { ! LOG (LOG_WARNING, "db->open failed on error %d: %s: %s", ! CTX->result, s->dictionary, db_strerror (CTX->result)); ! s->db = NULL; ! return EFILE; } if ((CTX->result = db_create (&s->sig, s->env, 0)) != 0) --- 1192,1233 ---- return EFAILURE; } ! if (fatal) ! { CTX->result = s->env->open (s->env, db_home, ! DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER_FATAL, 0); ! } ! else ! { CTX->result = s->env->open (s->env, db_home, DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER, 0); } ! if (CTX->result) ! { ! LOG (LOG_WARNING, "DB_ENV->open failed: %s", db_strerror (CTX->result)); ! return EFAILURE; } if ((CTX->result = db_create (&s->db, s->env, 0)) != 0) { LOG (LOG_WARNING, "db_create failed: %s", db_strerror (CTX->result)); s->db = NULL; ! return EFAILURE; } if ((CTX->result = s->db->DBOPEN (s->db, NULL, s->dictionary, NULL, DB_BTREE, dbflag, 0)) != 0) ! { ! LOG (LOG_WARNING, "db->open failed on error %d: %s: %s", ! CTX->result, s->dictionary, db_strerror (CTX->result)); ! s->db = NULL; ! return EFILE; } if ((CTX->result = db_create (&s->sig, s->env, 0)) != 0) *************** *** 1193,1249 **** s->sig->DBOPEN (s->sig, NULL, s->signature, NULL, DB_BTREE, DB_CREATE, 0)) != 0) { ! LOG (LOG_WARNING, "db->open failed on error %d: %s: %s", ! CTX->result, s->signature, db_strerror (CTX->result)); ! s->sig = NULL; ! _ds_shutdown_storage (CTX); ! return EFILE; } /* Obtain a lock for this user */ ! lock_result = _libdb3_drv_lock_get(CTX->username); ! if (lock_result < 0) { ! LOGDEBUG("locking subsystem returned error"); return EUNKNOWN; } ! ! if (lock_result > 0) { DBT dbt; int i = 0; ! LOGDEBUG("acquiring lock"); ! /* Run two instances of lock detection */ ! lock_detect(s->env, 0, DB_LOCK_OLDEST, NULL); ! lock_detect(s->env, 0, DB_LOCK_OLDEST, NULL); ! s->locker = 0; if ((CTX->result = lock_id (s->env, &s->locker)) != 0) { LOG (LOG_WARNING, "unable to get a locker id for %s: %s", ! CTX->username, db_strerror (CTX->result)); return EUNKNOWN; } ! dbt.data = CTX->username; dbt.size = strlen (CTX->username); while ((CTX->result = ! lock_get (s->env, s->locker, DB_LOCK_NOWAIT, &dbt, ! DB_LOCK_WRITE, &s->lock)) != 0 && i<15) { ! LOGDEBUG("spinning lock %d/15", i); i++; ! lock_detect(s->env, 0, DB_LOCK_OLDEST, NULL); ! lock_detect(s->env, 0, DB_LOCK_OLDEST, NULL); ! sleep(1); } ! ! if (CTX->result) { ! LOG (LOG_WARNING, ! "unable to obtain database lock for user %s: %s", ! CTX->username, db_strerror (CTX->result)); ! return EUNKNOWN; } } } --- 1242,1301 ---- s->sig->DBOPEN (s->sig, NULL, s->signature, NULL, DB_BTREE, DB_CREATE, 0)) != 0) { ! LOG (LOG_WARNING, "db->open failed on error %d: %s: %s", ! CTX->result, s->signature, db_strerror (CTX->result)); ! s->sig = NULL; ! _ds_shutdown_storage (CTX); ! return EFILE; } /* Obtain a lock for this user */ ! lock_result = _libdb3_drv_lock_get (CTX->username); ! if (lock_result < 0) ! { ! LOGDEBUG ("locking subsystem returned error"); return EUNKNOWN; } ! ! if (lock_result > 0) ! { DBT dbt; int i = 0; ! LOGDEBUG ("acquiring lock"); ! /* Run two instances of lock detection */ ! lock_detect (s->env, 0, DB_LOCK_OLDEST, NULL); ! lock_detect (s->env, 0, DB_LOCK_OLDEST, NULL); ! s->locker = 0; if ((CTX->result = lock_id (s->env, &s->locker)) != 0) { LOG (LOG_WARNING, "unable to get a locker id for %s: %s", ! CTX->username, db_strerror (CTX->result)); return EUNKNOWN; } ! dbt.data = CTX->username; dbt.size = strlen (CTX->username); while ((CTX->result = ! lock_get (s->env, s->locker, DB_LOCK_NOWAIT, &dbt, ! DB_LOCK_WRITE, &s->lock)) != 0 && i < 15) { ! LOGDEBUG ("spinning lock %d/15", i); i++; ! lock_detect (s->env, 0, DB_LOCK_OLDEST, NULL); ! lock_detect (s->env, 0, DB_LOCK_OLDEST, NULL); ! sleep (1); } ! ! if (CTX->result) ! { ! LOG (LOG_WARNING, ! "unable to obtain database lock for user %s: %s", ! CTX->username, db_strerror (CTX->result)); ! return EUNKNOWN; } } } *************** *** 1253,1259 **** s->sig = NULL; s->env = NULL; } ! s->c = NULL; s->dir_handles = nt_create (NT_INDEX); CTX->storage = s; --- 1305,1311 ---- s->sig = NULL; s->env = NULL; } ! s->c = NULL; s->dir_handles = nt_create (NT_INDEX); CTX->storage = s; *************** *** 1271,1277 **** memset (&CTX->totals, 0, sizeof (struct _ds_spam_totals)); } ! _ds_shutdown_storage(CTX); ! _ds_init_storage(CTX); return 0; } --- 1323,1329 ---- memset (&CTX->totals, 0, sizeof (struct _ds_spam_totals)); } ! _ds_shutdown_storage (CTX); ! _ds_init_storage (CTX); return 0; } Index: dspam/libdb4_drv.c diff -c dspam/libdb4_drv.c:1.1.1.1 dspam/libdb4_drv.c:1.1.1.1.2.1 *** dspam/libdb4_drv.c:1.1.1.1 Mon Nov 3 09:57:07 2003 --- dspam/libdb4_drv.c Sat Nov 15 18:34:58 2003 *************** *** 63,70 **** int dspam_init_driver (void) { ! _libdb4_drv_locks = nt_create(NT_PTR); ! if (_libdb4_drv_locks == NULL) return EUNKNOWN; return 0; --- 63,70 ---- int dspam_init_driver (void) { ! _libdb4_drv_locks = nt_create (NT_PTR); ! if (_libdb4_drv_locks == NULL) return EUNKNOWN; return 0; *************** *** 73,79 **** int dspam_shutdown_driver (void) { ! nt_destroy(_libdb4_drv_locks); return 0; } --- 73,79 ---- int dspam_shutdown_driver (void) { ! nt_destroy (_libdb4_drv_locks); return 0; } *************** *** 98,116 **** ret = (*s->db->get) (s->db, NULL, &key, &data, 0); if (ret) { ! if (ret == DB_RUNRECOVERY) { ! if (! _libdb4_drv_recover(CTX, 1)) { return _libdb4_drv_get_spamtotals (CTX); ! } else { return EUNKNOWN; } ! } else { LOGDEBUG ("_ds_get_totals: db->get failed: %s", db_strerror (ret)); return EFAILURE; } } ! memset(&CTX->totals, 0, sizeof(struct _ds_spam_totals)); memcpy (&CTX->totals, data.data, data.size); return 0; --- 98,122 ---- ret = (*s->db->get) (s->db, NULL, &key, &data, 0); if (ret) { ! if (ret == DB_RUNRECOVERY) ! { ! if (!_libdb4_drv_recover (CTX, 1)) ! { return _libdb4_drv_get_spamtotals (CTX); ! } ! else ! { return EUNKNOWN; } ! } ! else ! { LOGDEBUG ("_ds_get_totals: db->get failed: %s", db_strerror (ret)); return EFAILURE; } } ! memset (&CTX->totals, 0, sizeof (struct _ds_spam_totals)); memcpy (&CTX->totals, data.data, data.size); return 0; *************** *** 145,157 **** ret = (*s->db->put) (s->db, NULL, &key, &data, 0); if (ret) { ! if (ret == DB_RUNRECOVERY) { ! if (! _libdb4_drv_recover(CTX, 1)) { return _libdb4_drv_set_spamtotals (CTX); ! } else { return EUNKNOWN; } ! } else { LOGDEBUG ("_ds_set_totals: db->set failed: %s", db_strerror (ret)); return EFAILURE; } --- 151,169 ---- ret = (*s->db->put) (s->db, NULL, &key, &data, 0); if (ret) { ! if (ret == DB_RUNRECOVERY) ! { ! if (!_libdb4_drv_recover (CTX, 1)) ! { return _libdb4_drv_set_spamtotals (CTX); ! } ! else ! { return EUNKNOWN; } ! } ! else ! { LOGDEBUG ("_ds_set_totals: db->set failed: %s", db_strerror (ret)); return EFAILURE; } *************** *** 238,251 **** ret = (*s->db->get) (s->db, NULL, &key, &data, 0); ! if (ret) { ! if (ret == DB_RUNRECOVERY) { ! if (! _libdb4_drv_recover(CTX, 1)) { return _ds_get_spamrecord (CTX, token, stat); ! } else { return EUNKNOWN; } ! } else { return EFAILURE; } } --- 250,270 ---- ret = (*s->db->get) (s->db, NULL, &key, &data, 0); ! if (ret) ! { ! if (ret == DB_RUNRECOVERY) ! { ! if (!_libdb4_drv_recover (CTX, 1)) ! { return _ds_get_spamrecord (CTX, token, stat); ! } ! else ! { return EUNKNOWN; } ! } ! else ! { return EFAILURE; } } *************** *** 302,314 **** if (ret) { ! if (ret == DB_RUNRECOVERY) { ! if (! _libdb4_drv_recover(CTX, 1)) { return _ds_set_spamrecord (CTX, token, stat); ! } else { return EUNKNOWN; } ! } else { LOGDEBUG ("_ds_set_spamrecord: db->put failed: %s", db_strerror (ret)); return EFAILURE; } --- 321,339 ---- if (ret) { ! if (ret == DB_RUNRECOVERY) ! { ! if (!_libdb4_drv_recover (CTX, 1)) ! { return _ds_set_spamrecord (CTX, token, stat); ! } ! else ! { return EUNKNOWN; } ! } ! else ! { LOGDEBUG ("_ds_set_spamrecord: db->put failed: %s", db_strerror (ret)); return EFAILURE; } *************** *** 347,354 **** } else { ! LOGDEBUG("entering recovery mode"); ! free(CTX->storage); s = malloc (sizeof (struct _libdb4_drv_storage)); if (s == NULL) { --- 372,379 ---- } else { ! LOGDEBUG ("entering recovery mode"); ! free (CTX->storage); s = malloc (sizeof (struct _libdb4_drv_storage)); if (s == NULL) { *************** *** 356,367 **** LOG (LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } ! s->db = NULL; s->sig = NULL; s->env = NULL; ! _libdb4_drv_lock_set_readers((CTX->group == NULL) ? CTX->username : CTX->group, 0); } if (CTX->username != NULL) --- 381,393 ---- LOG (LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } ! s->db = NULL; s->sig = NULL; s->env = NULL; ! _libdb4_drv_lock_set_readers ((CTX->group == ! NULL) ? CTX->username : CTX->group, 0); } if (CTX->username != NULL) *************** *** 401,432 **** } */ ! if (!recovery) { CTX->result = ! s->env->open (s->env, db_home, DB_CREATE | DB_INIT_LOCK | ! DB_INIT_MPOOL, 0); ! } else if (recovery == 1) { CTX->result = s->env->open (s->env, db_home, ! DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER_FATAL, 0); ! } else { CTX->result = s->env->open (s->env, db_home, ! DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER, 0); } ! if (CTX->result) { if (recovery || CTX->result != DB_RUNRECOVERY) { ! LOG (LOG_WARNING, "DB_ENV->open failed: %s", db_strerror (CTX->result)); return EFAILURE; } else { ! recovery = 1; ! return _ds_init_storage (CTX); } } --- 427,465 ---- } */ ! if (!recovery) ! { CTX->result = ! s->env->open (s->env, db_home, DB_CREATE | DB_INIT_LOCK | ! DB_INIT_MPOOL, 0); ! } ! else if (recovery == 1) ! { CTX->result = s->env->open (s->env, db_home, ! DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER_FATAL, 0); ! } ! else ! { CTX->result = s->env->open (s->env, db_home, ! DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER, 0); } ! if (CTX->result) ! { if (recovery || CTX->result != DB_RUNRECOVERY) { ! LOG (LOG_WARNING, "DB_ENV->open failed: %s", ! db_strerror (CTX->result)); return EFAILURE; } else { ! recovery = 1; ! return _ds_init_storage (CTX); } } *************** *** 454,460 **** s->env->lock_put (s->env, &s->lock); s->env->lock_id_free (s->env, s->locker); ! s->env->close(s->env, 0); return _ds_init_storage (CTX); } --- 487,493 ---- s->env->lock_put (s->env, &s->lock); s->env->lock_id_free (s->env, s->locker); ! s->env->close (s->env, 0); return _ds_init_storage (CTX); } *************** *** 483,544 **** else { recovery = 1; ! s->db->close(s->db, 0); ! s->env->lock_put (s->env, &s->lock); s->env->lock_id_free (s->env, s->locker); ! s->env->close(s->env, 0); return _ds_init_storage (CTX); } } /* Obtain a lock for this user */ ! lock_result = _libdb4_drv_lock_get((CTX->group == NULL) ? CTX->username : CTX->group); ! if (lock_result < 0) { ! LOGDEBUG("locking subsystem returned error"); return EUNKNOWN; } ! ! if (lock_result > 0) { int i = 0; ! LOGDEBUG("acquiring lock"); /* Run two instances of lock detection */ ! s->env->lock_detect(s->env, 0, DB_LOCK_OLDEST, NULL); s->locker = 0; if ((CTX->result = s->env->lock_id (s->env, &s->locker)) != 0) { LOG (LOG_WARNING, "unable to get a locker id for %s: %s", ! CTX->username, db_strerror (CTX->result)); return EUNKNOWN; } dbt.data = CTX->username; dbt.size = strlen (CTX->username); CTX->result = ! s->env->lock_get (s->env, s->locker, DB_LOCK_NOWAIT, &dbt, ! DB_LOCK_WRITE, &s->lock); ! while(CTX->result != 0 && i<120) { ! LOGDEBUG("spinning lock %d/120", i); i++; ! s->env->lock_detect(s->env, 0, DB_LOCK_OLDEST, NULL); ! sleep(1); CTX->result = ! s->env->lock_get (s->env, s->locker, DB_LOCK_NOWAIT, &dbt, ! DB_LOCK_WRITE, &s->lock); } ! if (CTX->result) { ! if (!recovery) { recovery = 2; ! return _ds_init_storage(CTX); ! } else { LOG (LOG_WARNING, ! "unable to obtain database lock for user %s: %s", CTX->username, db_strerror (CTX->result)); return EUNKNOWN; } --- 516,586 ---- else { recovery = 1; ! s->db->close (s->db, 0); ! s->env->lock_put (s->env, &s->lock); s->env->lock_id_free (s->env, s->locker); ! s->env->close (s->env, 0); return _ds_init_storage (CTX); } } /* Obtain a lock for this user */ ! lock_result = ! _libdb4_drv_lock_get ((CTX->group == ! NULL) ? CTX->username : CTX->group); ! if (lock_result < 0) ! { ! LOGDEBUG ("locking subsystem returned error"); return EUNKNOWN; } ! ! if (lock_result > 0) ! { int i = 0; ! LOGDEBUG ("acquiring lock"); /* Run two instances of lock detection */ ! s->env->lock_detect (s->env, 0, DB_LOCK_OLDEST, NULL); s->locker = 0; if ((CTX->result = s->env->lock_id (s->env, &s->locker)) != 0) { LOG (LOG_WARNING, "unable to get a locker id for %s: %s", ! CTX->username, db_strerror (CTX->result)); return EUNKNOWN; } dbt.data = CTX->username; dbt.size = strlen (CTX->username); CTX->result = ! s->env->lock_get (s->env, s->locker, DB_LOCK_NOWAIT, &dbt, ! DB_LOCK_WRITE, &s->lock); ! while (CTX->result != 0 && i < 120) ! { ! LOGDEBUG ("spinning lock %d/120", i); i++; ! s->env->lock_detect (s->env, 0, DB_LOCK_OLDEST, NULL); ! sleep (1); CTX->result = ! s->env->lock_get (s->env, s->locker, DB_LOCK_NOWAIT, &dbt, ! DB_LOCK_WRITE, &s->lock); } ! if (CTX->result) ! { ! if (!recovery) ! { recovery = 2; ! return _ds_init_storage (CTX); ! } ! else ! { LOG (LOG_WARNING, ! "unable to obtain database lock for user %s: %s", CTX->username, db_strerror (CTX->result)); return EUNKNOWN; } *************** *** 569,578 **** memset (&CTX->totals, 0, sizeof (struct _ds_spam_totals)); } ! if (recovery) { recovery = 0; ! _ds_shutdown_storage(CTX); ! return _ds_init_storage(CTX); } return 0; } --- 611,621 ---- memset (&CTX->totals, 0, sizeof (struct _ds_spam_totals)); } ! if (recovery) ! { recovery = 0; ! _ds_shutdown_storage (CTX); ! return _ds_init_storage (CTX); } return 0; } *************** *** 612,625 **** { int lock_result; ! lock_result = _libdb4_drv_lock_put((CTX->group == NULL) ? CTX->username : CTX->group); ! if (lock_result < 0) { ! LOGDEBUG("locking subsystem returned error"); return EUNKNOWN; } ! if (lock_result > 0) { ! LOGDEBUG("freeing lock"); s->env->lock_put (s->env, &s->lock); s->env->lock_id_free (s->env, s->locker); } --- 655,672 ---- { int lock_result; ! lock_result = ! _libdb4_drv_lock_put ((CTX->group == ! NULL) ? CTX->username : CTX->group); ! if (lock_result < 0) ! { ! LOGDEBUG ("locking subsystem returned error"); return EUNKNOWN; } ! if (lock_result > 0) ! { ! LOGDEBUG ("freeing lock"); s->env->lock_put (s->env, &s->lock); s->env->lock_id_free (s->env, s->locker); } *************** *** 637,643 **** ret = s->env->close (s->env, 0); } ! if (!ret) { free (CTX->storage); CTX->storage = NULL; --- 684,690 ---- ret = s->env->close (s->env, 0); } ! if (!ret) { free (CTX->storage); CTX->storage = NULL; *************** *** 1058,1089 **** */ int ! _libdb4_drv_lock_get (const char *username) { struct nt_node *node; struct nt_c c; struct _libdb4_drv_lock *lock; ! node = c_nt_first(_libdb4_drv_locks, &c); ! while(node != NULL) { lock = (struct _libdb4_drv_lock *) node->ptr; ! if (lock != NULL && !strcmp(username, lock->username)) { lock->readers++; ! if (lock->readers==1) return 1; return 0; } ! node = c_nt_next(_libdb4_drv_locks, &c); } ! lock = malloc(sizeof(struct _libdb4_drv_lock)); ! if (lock == NULL) return EUNKNOWN; ! strcpy(lock->username, username); lock->readers = 1; ! nt_add(_libdb4_drv_locks, (void *) lock); return 1; } --- 1105,1138 ---- */ int ! _libdb4_drv_lock_get (const char *username) { struct nt_node *node; struct nt_c c; struct _libdb4_drv_lock *lock; ! node = c_nt_first (_libdb4_drv_locks, &c); ! while (node != NULL) ! { lock = (struct _libdb4_drv_lock *) node->ptr; ! if (lock != NULL && !strcmp (username, lock->username)) ! { lock->readers++; ! if (lock->readers == 1) return 1; return 0; } ! node = c_nt_next (_libdb4_drv_locks, &c); } ! lock = malloc (sizeof (struct _libdb4_drv_lock)); ! if (lock == NULL) return EUNKNOWN; ! strcpy (lock->username, username); lock->readers = 1; ! nt_add (_libdb4_drv_locks, (void *) lock); return 1; } *************** *** 1091,1107 **** 1: Free lock */ int ! _libdb4_drv_lock_put (const char *username) { struct nt_node *node; struct nt_c c; struct _libdb4_drv_lock *lock; ! ! node = c_nt_first(_libdb4_drv_locks, &c); ! while(node != NULL) { lock = (struct _libdb4_drv_lock *) node->ptr; ! if (lock != NULL && !strcmp(username, lock->username)) { ! if (lock->readers>0) { lock->readers--; if (lock->readers == 0) return 1; --- 1140,1159 ---- 1: Free lock */ int ! _libdb4_drv_lock_put (const char *username) { struct nt_node *node; struct nt_c c; struct _libdb4_drv_lock *lock; ! ! node = c_nt_first (_libdb4_drv_locks, &c); ! while (node != NULL) ! { lock = (struct _libdb4_drv_lock *) node->ptr; ! if (lock != NULL && !strcmp (username, lock->username)) ! { ! if (lock->readers > 0) ! { lock->readers--; if (lock->readers == 0) return 1; *************** *** 1109,1115 **** return 0; } ! node = c_nt_next(_libdb4_drv_locks, &c); } return EFAILURE; --- 1161,1167 ---- return 0; } ! node = c_nt_next (_libdb4_drv_locks, &c); } return EFAILURE; *************** *** 1123,1138 **** struct nt_c c; struct _libdb4_drv_lock *lock; ! node = c_nt_first(_libdb4_drv_locks, &c); ! while(node != NULL) { lock = (struct _libdb4_drv_lock *) node->ptr; ! if (lock != NULL && !strcmp(username, lock->username)) { ! lock->readers=readers; return 0; } ! node = c_nt_next(_libdb4_drv_locks, &c); } ! return EFAILURE; } --- 1175,1192 ---- struct nt_c c; struct _libdb4_drv_lock *lock; ! node = c_nt_first (_libdb4_drv_locks, &c); ! while (node != NULL) ! { lock = (struct _libdb4_drv_lock *) node->ptr; ! if (lock != NULL && !strcmp (username, lock->username)) ! { ! lock->readers = readers; return 0; } ! node = c_nt_next (_libdb4_drv_locks, &c); } ! return EFAILURE; } *************** *** 1142,1159 **** struct _libdb4_drv_storage *s = NULL; int dbflag = DB_CREATE; ! LOGDEBUG("recovering database"); ! if (CTX == NULL) return EINVAL; ! if (fatal) ! _libdb4_drv_lock_set_readers((CTX->group == NULL) ? CTX->username : CTX->group, 1); else ! _libdb4_drv_lock_set_readers((CTX->group == NULL) ? CTX->username : CTX->group, 0); ! _ds_shutdown_storage(CTX); ! _libdb4_drv_lock_set_readers((CTX->group == NULL) ? CTX->username : CTX->group, 0); s = malloc (sizeof (struct _libdb4_drv_storage)); if (s == NULL) --- 1196,1216 ---- struct _libdb4_drv_storage *s = NULL; int dbflag = DB_CREATE; ! LOGDEBUG ("recovering database"); ! if (CTX == NULL) return EINVAL; ! if (fatal) ! _libdb4_drv_lock_set_readers ((CTX->group == ! NULL) ? CTX->username : CTX->group, 1); else ! _libdb4_drv_lock_set_readers ((CTX->group == ! NULL) ? CTX->username : CTX->group, 0); ! _ds_shutdown_storage (CTX); ! _libdb4_drv_lock_set_readers ((CTX->group == ! NULL) ? CTX->username : CTX->group, 0); s = malloc (sizeof (struct _libdb4_drv_storage)); if (s == NULL) *************** *** 1162,1168 **** LOG (LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } ! s->db = NULL; s->sig = NULL; s->env = NULL; --- 1219,1225 ---- LOG (LOG_CRIT, ERROR_MEM_ALLOC); return EUNKNOWN; } ! s->db = NULL; s->sig = NULL; s->env = NULL; *************** *** 1170,1176 **** if (CTX->username != NULL) { char db_home[MAX_FILENAME_LENGTH]; ! if (CTX->group == NULL) { strcpy (s->dictionary, _ds_userdir_path (CTX->username, "dict")); --- 1227,1233 ---- if (CTX->username != NULL) { char db_home[MAX_FILENAME_LENGTH]; ! if (CTX->group == NULL) { strcpy (s->dictionary, _ds_userdir_path (CTX->username, "dict")); *************** *** 1193,1230 **** return EFAILURE; } ! if (fatal) { CTX->result = s->env->open (s->env, db_home, ! DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER_FATAL, 0); ! } else { CTX->result = s->env->open (s->env, db_home, DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER, 0); } ! if (CTX->result) { ! LOG (LOG_WARNING, "DB_ENV->open failed: %s", db_strerror (CTX->result)); ! return EFAILURE; } if ((CTX->result = db_create (&s->db, s->env, 0)) != 0) { LOG (LOG_WARNING, "db_create failed: %s", db_strerror (CTX->result)); s->db = NULL; ! return EFAILURE; } if ((CTX->result = s->db->DBOPEN (s->db, NULL, s->dictionary, NULL, DB_BTREE, dbflag, 0)) != 0) ! { ! LOG (LOG_WARNING, "db->open failed on error %d: %s: %s", ! CTX->result, s->dictionary, db_strerror (CTX->result)); ! s->db = NULL; ! return EFILE; } if ((CTX->result = db_create (&s->sig, s->env, 0)) != 0) --- 1250,1291 ---- return EFAILURE; } ! if (fatal) ! { CTX->result = s->env->open (s->env, db_home, ! DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER_FATAL, 0); ! } ! else ! { CTX->result = s->env->open (s->env, db_home, DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_RECOVER, 0); } ! if (CTX->result) ! { ! LOG (LOG_WARNING, "DB_ENV->open failed: %s", db_strerror (CTX->result)); ! return EFAILURE; } if ((CTX->result = db_create (&s->db, s->env, 0)) != 0) { LOG (LOG_WARNING, "db_create failed: %s", db_strerror (CTX->result)); s->db = NULL; ! return EFAILURE; } if ((CTX->result = s->db->DBOPEN (s->db, NULL, s->dictionary, NULL, DB_BTREE, dbflag, 0)) != 0) ! { ! LOG (LOG_WARNING, "db->open failed on error %d: %s: %s", ! CTX->result, s->dictionary, db_strerror (CTX->result)); ! s->db = NULL; ! return EFILE; } if ((CTX->result = db_create (&s->sig, s->env, 0)) != 0) *************** *** 1239,1249 **** s->sig->DBOPEN (s->sig, NULL, s->signature, NULL, DB_BTREE, DB_CREATE, 0)) != 0) { ! LOG (LOG_WARNING, "db->open failed on error %d: %s: %s", ! CTX->result, s->signature, db_strerror (CTX->result)); ! s->sig = NULL; ! _ds_shutdown_storage (CTX); ! return EFILE; } } --- 1300,1310 ---- s->sig->DBOPEN (s->sig, NULL, s->signature, NULL, DB_BTREE, DB_CREATE, 0)) != 0) { ! LOG (LOG_WARNING, "db->open failed on error %d: %s: %s", ! CTX->result, s->signature, db_strerror (CTX->result)); ! s->sig = NULL; ! _ds_shutdown_storage (CTX); ! return EFILE; } } *************** *** 1253,1264 **** s->sig = NULL; s->env = NULL; } ! s->c = NULL; s->dir_handles = nt_create (NT_INDEX); CTX->storage = s; ! _ds_shutdown_storage(CTX); ! _ds_init_storage(CTX); return 0; } --- 1314,1325 ---- s->sig = NULL; s->env = NULL; } ! s->c = NULL; s->dir_handles = nt_create (NT_INDEX); CTX->storage = s; ! _ds_shutdown_storage (CTX); ! _ds_init_storage (CTX); return 0; } Index: dspam/libdspam.c diff -c dspam/libdspam.c:1.1.1.13 dspam/libdspam.c:1.1.1.13.2.2 *** dspam/libdspam.c:1.1.1.13 Fri Oct 31 22:00:37 2003 --- dspam/libdspam.c Sat Nov 15 20:04:15 2003 *************** *** 95,100 **** --- 95,103 ---- if (!_ds_init_storage (CTX)) return CTX; + free (CTX->username); + free (CTX->group); + free (CTX); return NULL; } *************** *** 156,163 **** CTX->_process_start = time (NULL); ! if (CTX->mode != DSM_PROCESS && CTX->mode != DSM_CLASSIFY ! && (CTX->flags & DSF_SIGNATURE)) return _ds_process_signature (CTX); header = buffer_create (NULL); --- 159,166 ---- CTX->_process_start = time (NULL); ! if (CTX->mode != DSM_PROCESS && CTX->mode != DSM_CLASSIFY ! && (CTX->flags & DSF_SIGNATURE)) return _ds_process_signature (CTX); header = buffer_create (NULL); *************** *** 175,196 **** CTX->message = _ds_actualize_message (message); /* Skip this section is signature was provided for classification */ ! if (! (CTX->flags & DSF_SIGNATURE && CTX->mode == DSM_CLASSIFY)) { ! if (CTX->message == NULL) { LOG (LOG_WARNING, "_ds_actualize_message() failed"); return EUNKNOWN; } ! /* Iterate through each component and create large header/body buffers */ ! node_nt = c_nt_first (CTX->message->components, &c_nt); while (node_nt != NULL) { struct _ds_message_block *block = (struct _ds_message_block *) node_nt->ptr; ! #ifdef VERBOSE LOGDEBUG ("Processing component %d", i); #endif --- 178,200 ---- CTX->message = _ds_actualize_message (message); /* Skip this section is signature was provided for classification */ ! if (!(CTX->flags & DSF_SIGNATURE && CTX->mode == DSM_CLASSIFY && CTX->signature != NULL)) ! { ! if (CTX->message == NULL) { LOG (LOG_WARNING, "_ds_actualize_message() failed"); return EUNKNOWN; } ! /* Iterate through each component and create large header/body buffers */ ! node_nt = c_nt_first (CTX->message->components, &c_nt); while (node_nt != NULL) { struct _ds_message_block *block = (struct _ds_message_block *) node_nt->ptr; ! #ifdef VERBOSE LOGDEBUG ("Processing component %d", i); #endif *************** *** 198,204 **** if (block->media_type == MT_MULTIPART && block->media_subtype == MST_SIGNED) is_signed = 1; ! if (block->headers == NULL || block->headers->items == 0) { #ifdef VERBOSE --- 202,208 ---- if (block->media_type == MT_MULTIPART && block->media_subtype == MST_SIGNED) is_signed = 1; ! if (block->headers == NULL || block->headers->items == 0) { #ifdef VERBOSE *************** *** 210,216 **** else if (block->media_type == MT_TEXT || block->media_type == MT_MESSAGE || i == 0) { ! /* Accumulate the headers */ node_header = c_nt_first (block->headers, &c_nt2); while (node_header != NULL) --- 214,220 ---- else if (block->media_type == MT_TEXT || block->media_type == MT_MESSAGE || i == 0) { ! /* Accumulate the headers */ node_header = c_nt_first (block->headers, &c_nt2); while (node_header != NULL) *************** *** 218,247 **** struct _ds_header_field *current_header = (struct _ds_header_field *) node_header->ptr; snprintf (heading, sizeof (heading), ! "%s: %s\n", current_header->heading, current_header->data); buffer_cat (header, heading); node_header = c_nt_next (block->headers, &c_nt2); } ! decode = block->body->data; ! /* Accumulate the bodies */ if ((block->encoding == EN_BASE64 || block->encoding == EN_QUOTED_PRINTABLE) && (!is_signed || block-> ! original_signed_body ! == NULL)) { struct _ds_header_field *field; int is_attachment = 0; struct nt_node *node_hnt; struct nt_c c_hnt; ! node_hnt = c_nt_first (block->headers, &c_hnt); while (node_hnt != NULL) { field = (struct _ds_header_field *) node_hnt->ptr; ! if (field != NULL && field->heading != NULL && field->data != NULL) if (!strncasecmp (field->heading, "Content-Disposition", 19)) if (!strncasecmp (field->data, "attachment", 10)) is_attachment = 1; --- 222,253 ---- struct _ds_header_field *current_header = (struct _ds_header_field *) node_header->ptr; snprintf (heading, sizeof (heading), ! "%s: %s\n", current_header->heading, ! current_header->data); buffer_cat (header, heading); node_header = c_nt_next (block->headers, &c_nt2); } ! decode = block->body->data; ! /* Accumulate the bodies */ if ((block->encoding == EN_BASE64 || block->encoding == EN_QUOTED_PRINTABLE) && (!is_signed || block-> ! original_signed_body ! == NULL)) { struct _ds_header_field *field; int is_attachment = 0; struct nt_node *node_hnt; struct nt_c c_hnt; ! node_hnt = c_nt_first (block->headers, &c_hnt); while (node_hnt != NULL) { field = (struct _ds_header_field *) node_hnt->ptr; ! if (field != NULL && field->heading != NULL ! && field->data != NULL) if (!strncasecmp (field->heading, "Content-Disposition", 19)) if (!strncasecmp (field->data, "attachment", 10)) is_attachment = 1; *************** *** 262,268 **** buffer_cat (body, decode); if (decode != block->body->data) { ! if (is_signed) { LOGDEBUG --- 268,274 ---- buffer_cat (body, decode); if (decode != block->body->data) { ! if (is_signed) { LOGDEBUG *************** *** 275,284 **** block->encoding = EN_7BIT; buffer_destroy (block->body); } ! block->body = buffer_create (decode); free (decode); ! if (!is_signed) { /* Rewrite the actual header */ --- 281,290 ---- block->encoding = EN_7BIT; buffer_destroy (block->body); } ! block->body = buffer_create (decode); free (decode); ! if (!is_signed) { /* Rewrite the actual header */ *************** *** 287,293 **** { struct _ds_header_field *header = (struct _ds_header_field *) node_header->ptr; ! if (!strcasecmp (header->heading, "Content-Transfer-Encoding")) { free (header->data); header->data = strdup ("7bit"); --- 293,300 ---- { struct _ds_header_field *header = (struct _ds_header_field *) node_header->ptr; ! if (!strcasecmp ! (header->heading, "Content-Transfer-Encoding")) { free (header->data); header->data = strdup ("7bit"); *************** *** 411,417 **** */ int ! dspam_getsource (DSPAM_CTX * CTX, char *buf, size_t size, const char *whitelist) { struct _ds_message_block *current_block; struct _ds_header_field *current_heading = NULL; --- 418,425 ---- */ int ! dspam_getsource (DSPAM_CTX * CTX, char *buf, size_t size, ! const char *whitelist) { struct _ds_message_block *current_block; struct _ds_header_field *current_heading = NULL; *************** *** 443,469 **** { int whitelisted = 0; FILE *file; ! tok = strtok (NULL, "]"); ! if (tok != NULL) { ! if (!strcmp(tok, "127.0.0.1")) whitelisted = 1; ! if (whitelist != NULL) { ! file = fopen(whitelist, "r"); ! if (file != NULL) { char buffer[128]; ! while(fgets(buffer, sizeof(buffer), file)!=NULL) { ! chomp(buffer); ! LOGDEBUG("checking whitelist %s <> %s", tok, buffer); ! if (!strcmp(tok, buffer)) ! whitelisted = 1; } ! fclose(file); } } ! if (! whitelisted) { strlcpy (buf, tok, size); free (data); --- 451,481 ---- { int whitelisted = 0; FILE *file; ! tok = strtok (NULL, "]"); ! if (tok != NULL) ! { ! if (!strcmp (tok, "127.0.0.1")) whitelisted = 1; ! if (whitelist != NULL) ! { ! file = fopen (whitelist, "r"); ! if (file != NULL) ! { char buffer[128]; ! while (fgets (buffer, sizeof (buffer), file) != NULL) ! { ! chomp (buffer); ! LOGDEBUG ("checking whitelist %s <> %s", tok, buffer); ! if (!strcmp (tok, buffer)) ! whitelisted = 1; } ! fclose (file); } } ! if (!whitelisted) { strlcpy (buf, tok, size); free (data); *************** *** 510,532 **** /* Bayesian Calculations */ #if defined(BAYESIAN) || defined(DEBUG) float bay_top = 0.0; /* Standard Bayesian */ ! float bay_bot = 0.0; float bay_result = -1; long bay_used = 0; /* Total tokens used in bayes */ #endif #if defined(ALT_BAYESIAN) || defined(DEBUG) ! double abay_top = 0.0; /* Alternative Bayesian */ ! double abay_bot = 0.0; double abay_result = -1; ! long abay_used = 0; /* Total tokens used in altbayes */ #endif /* Chi-Square used to calculate confidence */ ! float chi_top = 0.0; /* Chi-Square */ float chi_bot = 0.0; float chi_result = -1; ! double p = 0.0, q = 0.0, s = 0.0; /* Chi-Calculations */ ! long chi_used = 0; /* Total tokens used in chi */ /* Long Hashed Token Tree: Track tokens, frequencies, and stats */ struct lht *freq = lht_create (1543); --- 522,544 ---- /* Bayesian Calculations */ #if defined(BAYESIAN) || defined(DEBUG) float bay_top = 0.0; /* Standard Bayesian */ ! float bay_bot = 0.0; float bay_result = -1; long bay_used = 0; /* Total tokens used in bayes */ #endif #if defined(ALT_BAYESIAN) || defined(DEBUG) ! double abay_top = 0.0; /* Alternative Bayesian */ ! double abay_bot = 0.0; double abay_result = -1; ! long abay_used = 0; /* Total tokens used in altbayes */ #endif /* Chi-Square used to calculate confidence */ ! float chi_top = 0.0; /* Chi-Square */ float chi_bot = 0.0; float chi_result = -1; ! double p = 0.0, q = 0.0, s = 0.0; /* Chi-Calculations */ ! long chi_used = 0; /* Total tokens used in chi */ /* Long Hashed Token Tree: Track tokens, frequencies, and stats */ struct lht *freq = lht_create (1543); *************** *** 537,545 **** struct tbt *index = tbt_create (); /* Binary tree index */ struct tbt_node *node_tbt; ! struct nt *header=NULL; /* header array */ struct nt_node *node_nt; struct nt_c c_nt; joined_token[0] = 0; --- 549,558 ---- struct tbt *index = tbt_create (); /* Binary tree index */ struct tbt_node *node_tbt; ! struct nt *header = NULL; /* header array */ struct nt_node *node_nt; struct nt_c c_nt; + struct _ds_spam_signature *input_sig = CTX->signature; joined_token[0] = 0; *************** *** 551,575 **** CTX->result = (CTX->mode == DSM_ADDSPAM) ? DSR_ISSPAM : DSR_ISINNOCENT; ! if (CTX->flags & DSF_SIGNATURE && CTX->mode == DSM_CLASSIFY) { ! int num_tokens = CTX->signature->length / sizeof (struct _ds_signature_token); struct _ds_signature_token t; ! for (i = 0; i < num_tokens; i++) { char x[128]; memcpy (&t, ! (char *) CTX->signature->data + ! (i * sizeof (struct _ds_signature_token)), sizeof (struct _ds_signature_token)); ! snprintf(x, sizeof(x), "E: %llu", t.token); lht_hit (freq, t.token, x); ! lht_setfrequency(freq, t.token, t.frequency); } } ! else { header = nt_create (NT_CHAR); ! if (header == NULL) { LOG (LOG_CRIT, ERROR_MEM_ALLOC); goto bail_unknown; } --- 564,593 ---- CTX->result = (CTX->mode == DSM_ADDSPAM) ? DSR_ISSPAM : DSR_ISINNOCENT; ! if (CTX->flags & DSF_SIGNATURE && CTX->mode == DSM_CLASSIFY && input_sig != NULL) ! { ! int num_tokens = ! input_sig->length / sizeof (struct _ds_signature_token); struct _ds_signature_token t; ! for (i = 0; i < num_tokens; i++) { char x[128]; memcpy (&t, ! (char *) input_sig->data + ! (i * sizeof (struct _ds_signature_token)), ! sizeof (struct _ds_signature_token)); ! snprintf (x, sizeof (x), "E: %llu", t.token); lht_hit (freq, t.token, x); ! lht_setfrequency (freq, t.token, t.frequency); } } ! else ! { header = nt_create (NT_CHAR); ! if (header == NULL) ! { LOG (LOG_CRIT, ERROR_MEM_ALLOC); goto bail_unknown; } *************** *** 589,595 **** int is_received; joined_token[0] = 0; alloc_joined = 0; ! line = node_nt->ptr; token = strtok (line, ":"); if (token != NULL && token[0] != 32 && token[0] != 9 --- 607,613 ---- int is_received; joined_token[0] = 0; alloc_joined = 0; ! line = node_nt->ptr; token = strtok (line, ":"); if (token != NULL && token[0] != 32 && token[0] != 9 *************** *** 604,610 **** #endif is_received = (!strcmp (heading, "Received") ? 1 : 0); ! if (is_received) token = strtok (NULL, DELIMITERS_HEADING); else --- 622,628 ---- #endif is_received = (!strcmp (heading, "Received") ? 1 : 0); ! if (is_received) token = strtok (NULL, DELIMITERS_HEADING); else *************** *** 613,619 **** while (token != NULL) { int l; ! l = strlen (token); if ((l > 2 && l < 25) || (l == 2 && !strchr (token, '$') && !strchr (token, '!'))) --- 631,637 ---- while (token != NULL) { int l; ! l = strlen (token); if ((l > 2 && l < 25) || (l == 2 && !strchr (token, '$') && !strchr (token, '!'))) *************** *** 630,636 **** { if (!_ds_process_header_token (CTX, joined_token, ! previous_token, freq, heading) && (CTX->flags & DSF_CHAINED)) { alloc_joined = 1; previous_token = strdup (joined_token); --- 648,655 ---- { if (!_ds_process_header_token (CTX, joined_token, ! previous_token, freq, heading) ! && (CTX->flags & DSF_CHAINED)) { alloc_joined = 1; previous_token = strdup (joined_token); *************** *** 650,670 **** } previous_token = token; } ! } else if (l == 1 || (l == 2 && (strchr (token, '$') || strchr (token, '!')))) { strlcat (joined_token, token, sizeof (joined_token)); } ! if (is_received) token = strtok (NULL, DELIMITERS_HEADING); else token = strtok (NULL, DELIMITERS); } node_nt = c_nt_next (header, &c_nt); ! if (joined_token[0] != 0) { if (strlen (joined_token) < 25 && joined_token[1] != 0) --- 669,689 ---- } previous_token = token; } ! } else if (l == 1 || (l == 2 && (strchr (token, '$') || strchr (token, '!')))) { strlcat (joined_token, token, sizeof (joined_token)); } ! if (is_received) token = strtok (NULL, DELIMITERS_HEADING); else token = strtok (NULL, DELIMITERS); } node_nt = c_nt_next (header, &c_nt); ! if (joined_token[0] != 0) { if (strlen (joined_token) < 25 && joined_token[1] != 0) *************** *** 673,679 **** previous_token, freq, heading); } } ! } nt_destroy (header); --- 692,698 ---- previous_token, freq, heading); } } ! } nt_destroy (header); *************** *** 691,717 **** char *url_ptr = url_body; int url_length; unsigned long long crc; ! token = strstr (url_ptr, "http://"); while (token != NULL) { url_ptr = token; ! token = strtok (token, " \n\">"); if (token != NULL) { url_length = strlen (token); ! /* Individual tokens form the URL */ token = strtok (token, DELIMITERS); while (token != NULL) { ! snprintf (combined_token, sizeof (combined_token), "Url*%s", token); crc = _ds_getcrc64 (combined_token); lht_hit (freq, crc, combined_token); token = strtok (NULL, DELIMITERS); } ! memset (body + (url_ptr - url_body), 32, url_length); url_ptr += url_length + 1; token = strstr (url_ptr, "http://"); --- 710,737 ---- char *url_ptr = url_body; int url_length; unsigned long long crc; ! token = strstr (url_ptr, "http://"); while (token != NULL) { url_ptr = token; ! token = strtok (token, " \n\">"); if (token != NULL) { url_length = strlen (token); ! /* Individual tokens form the URL */ token = strtok (token, DELIMITERS); while (token != NULL) { ! snprintf (combined_token, sizeof (combined_token), "Url*%s", ! token); crc = _ds_getcrc64 (combined_token); lht_hit (freq, crc, combined_token); token = strtok (NULL, DELIMITERS); } ! memset (body + (url_ptr - url_body), 32, url_length); url_ptr += url_length + 1; token = strstr (url_ptr, "http://"); *************** *** 721,727 **** } free (url_body); } ! url_body = strdup (body); if (url_body != NULL) { --- 741,747 ---- } free (url_body); } ! url_body = strdup (body); if (url_body != NULL) { *************** *** 729,768 **** char *url_ptr = url_body; int url_length; unsigned long long crc; ! url_ptr = url_body; token = strstr (url_ptr, "href=\""); while (token != NULL) { url_ptr = token + 6; ! token = strtok (url_ptr, " \n\">"); if (token != NULL) { url_length = strlen (token); ! /* Individual tokens form the URL */ token = strtok (url_ptr, DELIMITERS); while (token != NULL) { ! snprintf (combined_token, sizeof (combined_token), "Url*%s", token); crc = _ds_getcrc64 (combined_token); lht_hit (freq, crc, combined_token); token = strtok (NULL, DELIMITERS); } ! memset (body + (url_ptr - url_body), 32, url_length); ! url_ptr += url_length + 1; token = strstr (url_ptr, "href=\""); } else token = NULL; } ! free (url_body); } ! /* BODY: Split up the text into tokens, count frequency */ joined_token[0] = 0; alloc_joined = 0; --- 749,789 ---- char *url_ptr = url_body; int url_length; unsigned long long crc; ! url_ptr = url_body; token = strstr (url_ptr, "href=\""); while (token != NULL) { url_ptr = token + 6; ! token = strtok (url_ptr, " \n\">"); if (token != NULL) { url_length = strlen (token); ! /* Individual tokens form the URL */ token = strtok (url_ptr, DELIMITERS); while (token != NULL) { ! snprintf (combined_token, sizeof (combined_token), "Url*%s", ! token); crc = _ds_getcrc64 (combined_token); lht_hit (freq, crc, combined_token); token = strtok (NULL, DELIMITERS); } ! memset (body + (url_ptr - url_body), 32, url_length); ! url_ptr += url_length + 1; token = strstr (url_ptr, "href=\""); } else token = NULL; } ! free (url_body); } ! /* BODY: Split up the text into tokens, count frequency */ joined_token[0] = 0; alloc_joined = 0; *************** *** 788,794 **** } joined_token[0] = 0; } ! if (!_ds_process_body_token (CTX, token, previous_token, freq) && (CTX->flags & DSF_CHAINED)) { --- 809,815 ---- } joined_token[0] = 0; } ! if (!_ds_process_body_token (CTX, token, previous_token, freq) && (CTX->flags & DSF_CHAINED)) { *************** *** 807,813 **** } token = strtok (NULL, DELIMITERS); } ! if (joined_token[0] != 0) { if (strlen (joined_token) < 25 && joined_token[1] != 0) --- 828,834 ---- } token = strtok (NULL, DELIMITERS); } ! if (joined_token[0] != 0) { if (strlen (joined_token) < 25 && joined_token[1] != 0) *************** *** 829,835 **** while (node_lht != NULL) { _ds_calc_stat (CTX, node_lht->key, &node_lht->s); ! tbt_add (index, node_lht->s.probability, node_lht->key, node_lht->frequency); #ifdef VERBOSE LOGDEBUG ("Token: %s [%f]", node_lht->token_name, --- 850,857 ---- while (node_lht != NULL) { _ds_calc_stat (CTX, node_lht->key, &node_lht->s); ! tbt_add (index, node_lht->s.probability, node_lht->key, ! node_lht->frequency); #ifdef VERBOSE LOGDEBUG ("Token: %s [%f]", node_lht->token_name, *************** *** 850,856 **** } /* Initialize signature if requested */ ! if (CTX->flags & DSF_SIGNATURE) { CTX->signature = calloc (1, sizeof (struct _ds_spam_signature)); if (CTX->signature == NULL) --- 872,878 ---- } /* Initialize signature if requested */ ! if (CTX->flags & DSF_SIGNATURE && (CTX->mode != DSM_CLASSIFY || input_sig == NULL)) { CTX->signature = calloc (1, sizeof (struct _ds_spam_signature)); if (CTX->signature == NULL) *************** *** 861,867 **** return EUNKNOWN; } ! CTX->signature->length = sizeof (struct _ds_signature_token) * index->items; CTX->signature->data = malloc (CTX->signature->length); if (CTX->signature->data == NULL) { --- 883,890 ---- return EUNKNOWN; } ! CTX->signature->length = ! sizeof (struct _ds_signature_token) * index->items; CTX->signature->data = malloc (CTX->signature->length); if (CTX->signature->data == NULL) { *************** *** 876,882 **** node_tbt = tbt_first (index); ! for(i=0; i<((index->items > 27) ? 27 : index->items); i++) { unsigned long long crc; char *token_name; --- 899,905 ---- node_tbt = tbt_first (index); ! for (i = 0; i < ((index->items > 27) ? 27 : index->items); i++) { unsigned long long crc; char *token_name; *************** *** 902,925 **** /* Bayesian */ #if defined(BAYESIAN) || defined(DEBUG) ! if (bay_used < 15) { ! #ifdef BAYESIAN #ifndef VERBOSE ! if (CTX->mode != DSM_CLASSIFY) { #endif ! LOGDEBUG ("[bayes] combining: [%2.6f] %s (%df, %lds, %ldi)", ! stat.probability, token_name, lht_getfrequency(freq, crc), ! stat.spam_hits, stat.innocent_hits); #ifndef VERBOSE } #endif #endif ! if (bay_used == 0) { bay_top = stat.probability; bay_bot = 1 - stat.probability; ! } else { bay_top *= stat.probability; bay_bot *= (1 - stat.probability); } --- 925,953 ---- /* Bayesian */ #if defined(BAYESIAN) || defined(DEBUG) ! if (bay_used < 15) ! { ! #ifdef BAYESIAN #ifndef VERBOSE ! if (CTX->mode != DSM_CLASSIFY) ! { #endif ! LOGDEBUG ("[bayes] combining: [%2.6f] %s (%df, %lds, %ldi)", ! stat.probability, token_name, lht_getfrequency (freq, crc), ! stat.spam_hits, stat.innocent_hits); #ifndef VERBOSE } #endif #endif ! if (bay_used == 0) ! { bay_top = stat.probability; bay_bot = 1 - stat.probability; ! } ! else ! { bay_top *= stat.probability; bay_bot *= (1 - stat.probability); } *************** *** 930,963 **** /* Alternative Bayesian */ #if defined(ALT_BAYESIAN) || defined(DEBUG) ! if (abay_used < 27) { ! #ifdef ALT_BAYESIAN ! if (CTX->mode != DSM_CLASSIFY) { LOGDEBUG ("[altbayes] combining: [%2.6f] %s (%df, %lds, %ldi)", ! stat.probability, token_name, lht_getfrequency(freq, crc), ! stat.spam_hits, stat.innocent_hits); } #endif ! ! if (abay_used == 0) { abay_top = stat.probability; abay_bot = (1 - stat.probability); ! } else { abay_top *= stat.probability; abay_bot *= (1 - stat.probability); } abay_used++; ! if (abay_used < 27 && lht_getfrequency (freq, crc)>1) { #ifdef ALT_BAYESIAN ! if (CTX->mode != DSM_CLASSIFY) { LOGDEBUG ("[altbayes] combining: [%2.6f] %s (%df, %lds, %ldi)", ! stat.probability, token_name, lht_getfrequency(freq, crc), ! stat.spam_hits, stat.innocent_hits); } #endif ! abay_used++; abay_top *= stat.probability; abay_bot *= (1 - stat.probability); } --- 958,999 ---- /* Alternative Bayesian */ #if defined(ALT_BAYESIAN) || defined(DEBUG) ! if (abay_used < 27) ! { ! #ifdef ALT_BAYESIAN ! if (CTX->mode != DSM_CLASSIFY) ! { LOGDEBUG ("[altbayes] combining: [%2.6f] %s (%df, %lds, %ldi)", ! stat.probability, token_name, lht_getfrequency (freq, crc), ! stat.spam_hits, stat.innocent_hits); } #endif ! ! if (abay_used == 0) ! { abay_top = stat.probability; abay_bot = (1 - stat.probability); ! } ! else ! { abay_top *= stat.probability; abay_bot *= (1 - stat.probability); } abay_used++; ! if (abay_used < 27 && lht_getfrequency (freq, crc) > 1) ! { #ifdef ALT_BAYESIAN ! if (CTX->mode != DSM_CLASSIFY) ! { LOGDEBUG ("[altbayes] combining: [%2.6f] %s (%df, %lds, %ldi)", ! stat.probability, token_name, lht_getfrequency (freq, ! crc), ! stat.spam_hits, stat.innocent_hits); } #endif ! abay_used++; abay_top *= stat.probability; abay_bot *= (1 - stat.probability); } *************** *** 967,974 **** /* Chi-Square */ /* Definitions used in BogoFilter */ ! #define CHI_S 0.010 /* Sensitivity */ ! #define CHI_X 0.415 /* Value to use when N = 0 */ #define CHI_CUTOFF 0.54 if (chi_used < 25) --- 1003,1010 ---- /* Chi-Square */ /* Definitions used in BogoFilter */ ! #define CHI_S 0.010 /* Sensitivity */ ! #define CHI_X 0.415 /* Value to use when N = 0 */ #define CHI_CUTOFF 0.54 if (chi_used < 25) *************** *** 976,1010 **** float probability; long n = (index->items > 25) ? 25 : index->items; ! probability = ((CHI_S * CHI_X) + (n * stat.probability)) / ! (CHI_S+ n); #ifdef CHI_SQUARE ! if (CTX->mode != DSM_CLASSIFY) { LOGDEBUG ("[chi-sq] combining: [%2.6f] %s (%df, %lds, %ldi)", ! stat.probability, token_name, lht_getfrequency(freq, crc), ! stat.spam_hits, stat.innocent_hits); } #endif ! if (probability < 0.3 || probability > 0.7) { ! if (chi_used == 0) { chi_top = probability; chi_bot = (1 - probability); ! } else { chi_top *= probability; chi_bot *= (1 - probability); } chi_used++; ! if (chi_used < 25 && lht_getfrequency (freq, crc)>1) { #ifdef CHI_SQUARE ! if (CTX->mode != DSM_CLASSIFY) { LOGDEBUG ("[chi-sq] combining: [%2.6f] %s (%df, %lds, %ldi)", ! stat.probability, token_name, lht_getfrequency(freq, crc), ! stat.spam_hits, stat.innocent_hits); } #endif --- 1012,1053 ---- float probability; long n = (index->items > 25) ? 25 : index->items; ! probability = ((CHI_S * CHI_X) + (n * stat.probability)) / (CHI_S + n); #ifdef CHI_SQUARE ! if (CTX->mode != DSM_CLASSIFY) ! { LOGDEBUG ("[chi-sq] combining: [%2.6f] %s (%df, %lds, %ldi)", ! stat.probability, token_name, lht_getfrequency (freq, crc), ! stat.spam_hits, stat.innocent_hits); } #endif ! if (probability < 0.3 || probability > 0.7) ! { ! if (chi_used == 0) ! { chi_top = probability; chi_bot = (1 - probability); ! } ! else ! { chi_top *= probability; chi_bot *= (1 - probability); } chi_used++; ! if (chi_used < 25 && lht_getfrequency (freq, crc) > 1) ! { #ifdef CHI_SQUARE ! if (CTX->mode != DSM_CLASSIFY) ! { LOGDEBUG ("[chi-sq] combining: [%2.6f] %s (%df, %lds, %ldi)", ! stat.probability, token_name, lht_getfrequency (freq, ! crc), ! stat.spam_hits, stat.innocent_hits); } #endif *************** *** 1029,1041 **** #if defined(ALT_BAYESIAN) || defined(DEBUG) abay_result = (abay_top) / (abay_top + abay_bot); ! LOGDEBUG ("Alternative Bayes Probability: %f Samples: %ld", abay_result, abay_used); #endif /* Chi-Square */ ! if (chi_used == 0) { p = q = s = 0; ! } else { p = 1.0 - pow (chi_bot, 1.0 / chi_used); q = 1.0 - pow (chi_top, 1.0 / chi_used); s = (p - q) / (p + q); --- 1072,1088 ---- #if defined(ALT_BAYESIAN) || defined(DEBUG) abay_result = (abay_top) / (abay_top + abay_bot); ! LOGDEBUG ("Alternative Bayes Probability: %f Samples: %ld", abay_result, ! abay_used); #endif /* Chi-Square */ ! if (chi_used == 0) ! { p = q = s = 0; ! } ! else ! { p = 1.0 - pow (chi_bot, 1.0 / chi_used); q = 1.0 - pow (chi_top, 1.0 / chi_used); s = (p - q) / (p + q); *************** *** 1045,1051 **** chi_result = s; #if defined(CHI_SQUARE) || defined(DEBUG) ! LOGDEBUG ("Chi-Square Confidence: %f (Spamminess: %f, Non-Spamminess: %f, Samples: %ld)", chi_result, p, q, chi_used); #endif /* END Calculate Individual Probabilities */ --- 1092,1100 ---- chi_result = s; #if defined(CHI_SQUARE) || defined(DEBUG) ! LOGDEBUG ! ("Chi-Square Confidence: %f (Spamminess: %f, Non-Spamminess: %f, Samples: %ld)", ! chi_result, p, q, chi_used); #endif /* END Calculate Individual Probabilities */ *************** *** 1053,1081 **** /* BEGIN Determine Result */ #ifdef BAYESIAN ! if (bay_result >= 0.9 && CTX->mode != DSM_FALSEPOSITIVE) { CTX->result = DSR_ISSPAM; CTX->probability = bay_result; } #endif #ifdef ALT_BAYESIAN ! if (abay_result >= 0.9 && CTX->mode != DSM_FALSEPOSITIVE) { CTX->result = DSR_ISSPAM; CTX->probability = abay_result; } #endif #ifdef CHI_SQUARE ! if (chi_result >= CHI_CUTOFF && CTX->mode != DSM_FALSEPOSITIVE) { CTX->result = DSR_ISSPAM; ! if (CTX->probability < 0) CTX->probability = chi_result; } #endif /* If the message is innocent, we'll need to assign one probability */ ! if (CTX->probability < 0) { #ifdef BAYESIAN CTX->probability = bay_result; #else --- 1102,1134 ---- /* BEGIN Determine Result */ #ifdef BAYESIAN ! if (bay_result >= 0.9 && CTX->mode != DSM_FALSEPOSITIVE) ! { CTX->result = DSR_ISSPAM; CTX->probability = bay_result; } #endif #ifdef ALT_BAYESIAN ! if (abay_result >= 0.9 && CTX->mode != DSM_FALSEPOSITIVE) ! { CTX->result = DSR_ISSPAM; CTX->probability = abay_result; } #endif #ifdef CHI_SQUARE ! if (chi_result >= CHI_CUTOFF && CTX->mode != DSM_FALSEPOSITIVE) ! { CTX->result = DSR_ISSPAM; ! if (CTX->probability < 0) CTX->probability = chi_result; } #endif /* If the message is innocent, we'll need to assign one probability */ ! if (CTX->probability < 0) ! { #ifdef BAYESIAN CTX->probability = bay_result; #else *************** *** 1090,1131 **** /* END Determine Result */ #ifdef DEBUG ! if ( abay_result >= 0.9 && bay_result < 0.9) { ! LOGDEBUG("CATCH: Alternative Bayesian"); ! } else if ( abay_result < 0.9 && bay_result >= 0.9) { ! LOGDEBUG("MISS: Alternative Bayesian"); } ! if (chi_result >= CHI_CUTOFF && bay_result < 0.9) { ! LOGDEBUG("CATCH: Chi-Square"); ! } else if (chi_result < CHI_CUTOFF && bay_result >= 0.9) { ! LOGDEBUG("MISS: Chi-Square"); } #endif /* Update Totals */ /* SPAM */ ! if (CTX->result == DSR_ISSPAM) { CTX->totals.total_spam++; ! if (CTX->mode == DSM_ADDSPAM) { if (CTX->flags & DSF_CORPUS || CTX->flags & DSF_INOCULATE) CTX->totals.spam_corpusfed++; ! else { CTX->totals.spam_misclassified++; ! CTX->totals.total_innocent -= (CTX->totals.total_innocent > 0) ? 1 : 0; } } ! /* INNOCENT */ ! } else if (CTX->result == DSR_ISINNOCENT) { CTX->totals.total_innocent++; ! if (CTX->flags & DSF_CORPUS || CTX->flags & DSF_INOCULATE) { ! CTX->totals.innocent_corpusfed++; ! } else if (CTX->mode == DSM_FALSEPOSITIVE) { CTX->totals.innocent_misclassified++; CTX->totals.total_spam -= (CTX->totals.total_spam > 0) ? 1 : 0; } --- 1143,1199 ---- /* END Determine Result */ #ifdef DEBUG ! if (abay_result >= 0.9 && bay_result < 0.9) ! { ! LOGDEBUG ("CATCH: Alternative Bayesian"); ! } ! else if (abay_result < 0.9 && bay_result >= 0.9) ! { ! LOGDEBUG ("MISS: Alternative Bayesian"); } ! if (chi_result >= CHI_CUTOFF && bay_result < 0.9) ! { ! LOGDEBUG ("CATCH: Chi-Square"); ! } ! else if (chi_result < CHI_CUTOFF && bay_result >= 0.9) ! { ! LOGDEBUG ("MISS: Chi-Square"); } #endif /* Update Totals */ /* SPAM */ ! if (CTX->result == DSR_ISSPAM) ! { CTX->totals.total_spam++; ! if (CTX->mode == DSM_ADDSPAM) ! { if (CTX->flags & DSF_CORPUS || CTX->flags & DSF_INOCULATE) CTX->totals.spam_corpusfed++; ! else ! { CTX->totals.spam_misclassified++; ! CTX->totals.total_innocent -= ! (CTX->totals.total_innocent > 0) ? 1 : 0; } } ! /* INNOCENT */ ! } ! else if (CTX->result == DSR_ISINNOCENT) ! { CTX->totals.total_innocent++; ! if (CTX->flags & DSF_CORPUS || CTX->flags & DSF_INOCULATE) ! { ! CTX->totals.innocent_corpusfed++; ! } ! else if (CTX->mode == DSM_FALSEPOSITIVE) ! { CTX->totals.innocent_misclassified++; CTX->totals.total_spam -= (CTX->totals.total_spam > 0) ? 1 : 0; } *************** *** 1134,1140 **** node_tbt = tbt_first (index); i = 0; ! while(node_tbt != NULL) { unsigned long long crc; char *token_name; --- 1202,1209 ---- node_tbt = tbt_first (index); i = 0; ! while (node_tbt != NULL) ! { unsigned long long crc; char *token_name; *************** *** 1142,1158 **** crc = node_tbt->token; token_name = lht_gettoken (freq, crc); ! if (lht_getspamstat (freq, crc, &stat) || token_name == NULL) { node_tbt = tbt_next (node_tbt); continue; } ! if (CTX->flags & DSF_SIGNATURE) { struct _ds_signature_token t; t.token = crc; ! t.frequency = lht_getfrequency(freq, t.token); memcpy ((char *) CTX->signature->data + (i * sizeof (struct _ds_signature_token)), &t, sizeof (struct _ds_signature_token)); --- 1211,1228 ---- crc = node_tbt->token; token_name = lht_gettoken (freq, crc); ! if (lht_getspamstat (freq, crc, &stat) || token_name == NULL) ! { node_tbt = tbt_next (node_tbt); continue; } ! if (CTX->flags & DSF_SIGNATURE && (CTX->mode != DSM_CLASSIFY || input_sig == NULL)) { struct _ds_signature_token t; t.token = crc; ! t.frequency = lht_getfrequency (freq, t.token); memcpy ((char *) CTX->signature->data + (i * sizeof (struct _ds_signature_token)), &t, sizeof (struct _ds_signature_token)); *************** *** 1166,1178 **** stat.probability = 0.00; } ! if (CTX->result == DSR_ISSPAM) { ! if (CTX->flags & DSF_INOCULATE) { if (stat.innocent_hits < 2 && stat.spam_hits < 5) stat.spam_hits += 5; else stat.spam_hits += 2; ! } else { stat.spam_hits++; } } --- 1236,1252 ---- stat.probability = 0.00; } ! if (CTX->result == DSR_ISSPAM) ! { ! if (CTX->flags & DSF_INOCULATE) ! { if (stat.innocent_hits < 2 && stat.spam_hits < 5) stat.spam_hits += 5; else stat.spam_hits += 2; ! } ! else ! { stat.spam_hits++; } } *************** *** 1183,1189 **** if (CTX->mode == DSM_ADDSPAM && (!(CTX->flags & DSF_CORPUS))) stat.innocent_hits--; ! if (CTX->mode == DSM_FALSEPOSITIVE) { stat.innocent_hits++; stat.spam_hits--; --- 1257,1264 ---- if (CTX->mode == DSM_ADDSPAM && (!(CTX->flags & DSF_CORPUS))) stat.innocent_hits--; ! if (CTX->mode == DSM_FALSEPOSITIVE) ! { stat.innocent_hits++; stat.spam_hits--; *************** *** 1208,1225 **** tbt_destroy (index); lht_destroy (freq); ! if (CTX->mode == DSM_FALSEPOSITIVE) { CTX->probability = 0.0; CTX->result = DSR_ISINNOCENT; } ! else if (CTX->mode == DSM_ADDSPAM) { CTX->probability = 1.0; CTX->result = DSR_ISSPAM; } ! if (CTX->result == DSR_ISSPAM) { CTX->confidence = chi_result; ! } else { CTX->confidence = 1.0 - chi_result; } --- 1283,1305 ---- tbt_destroy (index); lht_destroy (freq); ! if (CTX->mode == DSM_FALSEPOSITIVE) ! { CTX->probability = 0.0; CTX->result = DSR_ISINNOCENT; } ! else if (CTX->mode == DSM_ADDSPAM) ! { CTX->probability = 1.0; CTX->result = DSR_ISSPAM; } ! if (CTX->result == DSR_ISSPAM) ! { CTX->confidence = chi_result; ! } ! else ! { CTX->confidence = 1.0 - chi_result; } *************** *** 1263,1281 **** CTX->totals.total_spam -= (CTX->totals.total_spam > 0) ? 1 : 0; CTX->totals.total_innocent++; ! } else if (CTX->mode == DSM_ADDSPAM) { if (CTX->flags & DSF_CORPUS || CTX->flags & DSF_INOCULATE) CTX->totals.spam_corpusfed++; ! else { CTX->totals.spam_misclassified++; CTX->totals.total_innocent -= (CTX->totals.total_innocent > 0) ? 1 : 0; } CTX->totals.total_spam++; ! } else { /* If it's not a false positive, forwarded spam, or corpus-added spam, ! it's a corpus-added innocent */ if (CTX->flags & DSF_CORPUS || CTX->flags & DSF_INOCULATE) CTX->totals.innocent_corpusfed++; --- 1343,1366 ---- CTX->totals.total_spam -= (CTX->totals.total_spam > 0) ? 1 : 0; CTX->totals.total_innocent++; ! } ! else if (CTX->mode == DSM_ADDSPAM) ! { if (CTX->flags & DSF_CORPUS || CTX->flags & DSF_INOCULATE) CTX->totals.spam_corpusfed++; ! else ! { CTX->totals.spam_misclassified++; CTX->totals.total_innocent -= (CTX->totals.total_innocent > 0) ? 1 : 0; } CTX->totals.total_spam++; ! } ! else ! { /* If it's not a false positive, forwarded spam, or corpus-added spam, ! * it's a corpus-added innocent */ if (CTX->flags & DSF_CORPUS || CTX->flags & DSF_INOCULATE) CTX->totals.innocent_corpusfed++; *************** *** 1290,1296 **** { memcpy (&t, (char *) CTX->signature->data + ! (i * sizeof (struct _ds_signature_token)), sizeof (struct _ds_signature_token)); lht_hit (freq, t.token, "-"); lht_setfrequency (freq, t.token, t.frequency); } --- 1375,1382 ---- { memcpy (&t, (char *) CTX->signature->data + ! (i * sizeof (struct _ds_signature_token)), ! sizeof (struct _ds_signature_token)); lht_hit (freq, t.token, "-"); lht_setfrequency (freq, t.token, t.frequency); } *************** *** 1310,1321 **** { if (!(CTX->flags & DSF_CORPUS)) node_lht->s.innocent_hits -= (node_lht->s.innocent_hits > 0) ? 1 : 0; ! if (CTX->flags & DSF_INOCULATE) { if (node_lht->s.innocent_hits < 2 && node_lht->s.spam_hits < 5) node_lht->s.spam_hits += 5; ! else node_lht->s.spam_hits += 2; ! } else { node_lht->s.spam_hits++; } } --- 1396,1410 ---- { if (!(CTX->flags & DSF_CORPUS)) node_lht->s.innocent_hits -= (node_lht->s.innocent_hits > 0) ? 1 : 0; ! if (CTX->flags & DSF_INOCULATE) ! { if (node_lht->s.innocent_hits < 2 && node_lht->s.spam_hits < 5) node_lht->s.spam_hits += 5; ! else node_lht->s.spam_hits += 2; ! } ! else ! { node_lht->s.spam_hits++; } } *************** *** 1366,1372 **** s->probability = .4; if (CTX->totals.total_spam > 0 && CTX->totals.total_innocent > 0 && ! ((s->spam_hits * 1.0 / CTX->totals.total_spam * 1.0) + (s->innocent_hits * 1.0 / CTX->totals.total_innocent * 1.0))>0) { #ifdef BIAS s->probability = --- 1455,1462 ---- s->probability = .4; if (CTX->totals.total_spam > 0 && CTX->totals.total_innocent > 0 && ! ((s->spam_hits * 1.0 / CTX->totals.total_spam * 1.0) + ! (s->innocent_hits * 1.0 / CTX->totals.total_innocent * 1.0)) > 0) { #ifdef BIAS s->probability = Index: dspam/libdspam.h diff -c dspam/libdspam.h:1.1.1.4 dspam/libdspam.h:1.1.1.4.2.1 *** dspam/libdspam.h:1.1.1.4 Mon Oct 27 11:37:16 2003 --- dspam/libdspam.h Sat Nov 15 18:34:58 2003 *************** *** 31,37 **** int dspam_process (DSPAM_CTX * CTX, const char *message); int dspam_destroy (DSPAM_CTX * CTX); int dspam_getresult (DSPAM_CTX * CTX); ! int dspam_getsource (DSPAM_CTX * CTX, char *buf, size_t size, const char *whitelist); /* DSPAM's private functions */ int _ds_calc_stat (DSPAM_CTX * CTX, unsigned long long token, --- 31,38 ---- int dspam_process (DSPAM_CTX * CTX, const char *message); int dspam_destroy (DSPAM_CTX * CTX); int dspam_getresult (DSPAM_CTX * CTX); ! int dspam_getsource (DSPAM_CTX * CTX, char *buf, size_t size, ! const char *whitelist); /* DSPAM's private functions */ int _ds_calc_stat (DSPAM_CTX * CTX, unsigned long long token, *************** *** 55,61 **** /* Return Codes */ #ifndef EINVAL ! # define EINVAL -0x01 /* Invalid call or parms - Already Initialized or shutdown */ #endif #define EUNKNOWN -0x02 /* Unknown/unexpected error */ --- 56,62 ---- /* Return Codes */ #ifndef EINVAL ! # define EINVAL -0x01 /* Invalid call or parms - Already Initialized or shutdown */ #endif #define EUNKNOWN -0x02 /* Unknown/unexpected error */ Index: dspam/libdspam_objects.h diff -c dspam/libdspam_objects.h:1.1.1.6 dspam/libdspam_objects.h:1.1.1.6.2.1 *** dspam/libdspam_objects.h:1.1.1.6 Wed Oct 29 05:14:46 2003 --- dspam/libdspam_objects.h Sat Nov 15 18:34:58 2003 *************** *** 67,73 **** time_t _process_start; /* Process start time */ float probability; /* DSPAM Probability Result */ int result; /* DSPAM Boolean Result */ ! float confidence; /* DSPAM Confidence in Result */ int mode; /* DSPAM Processor mode */ int flags; /* DSPAM Processor flags */ int locked; /* Is user locked? */ --- 67,73 ---- time_t _process_start; /* Process start time */ float probability; /* DSPAM Probability Result */ int result; /* DSPAM Boolean Result */ ! float confidence; /* DSPAM Confidence in Result */ int mode; /* DSPAM Processor mode */ int flags; /* DSPAM Processor flags */ int locked; /* Is user locked? */ *************** *** 79,85 **** /* DSPAM Flags */ #define DSF_CHAINED 0x01 /* Use chained tokens */ #define DSF_SIGNATURE 0x02 /* Signature provided/requested */ ! #define DSF_INOCULATE 0x04 /* Inoculate user */ #define DSF_COPYBACK 0x08 /* Copy back decoded messages */ #define DSF_IGNOREHEADER 0x10 /* Ignore message headers */ #define DSF_CORPUS 0x20 /* Is corpus mail */ --- 79,85 ---- /* DSPAM Flags */ #define DSF_CHAINED 0x01 /* Use chained tokens */ #define DSF_SIGNATURE 0x02 /* Signature provided/requested */ ! #define DSF_INOCULATE 0x04 /* Inoculate user */ #define DSF_COPYBACK 0x08 /* Copy back decoded messages */ #define DSF_IGNOREHEADER 0x10 /* Ignore message headers */ #define DSF_CORPUS 0x20 /* Is corpus mail */ Index: dspam/maketest diff -c /dev/null dspam/maketest:1.3.2.1 *** /dev/null Sat Nov 15 20:45:12 2003 --- dspam/maketest Sat Nov 15 18:40:36 2003 *************** *** 0 **** --- 1,14 ---- + LIBDSPAM = .libs/libdspam.a + + run: testlibdspam + ./testlibdspam + + testutil.o: util.c + gcc -c -g -o testutil.o -I. -DHAVE_CONFIG_H -DUSERDIR=\"/tmp\" util.c + + testerror.o: error.c + gcc -c -g -o testerror.o -I. -DHAVE_CONFIG_H -DUSERDIR=\"/tmp\" error.c + + testlibdspam: testlibdspam.c testutil.o testerror.o $(LIBDSPAM) + gcc -g -o testlibdspam testlibdspam.c \ + testutil.o testerror.o $(LIBDSPAM) -ldb -lcheck -lm Index: dspam/md5.c diff -c dspam/md5.c:1.1.1.1 dspam/md5.c:1.1.1.1.2.1 *** dspam/md5.c:1.1.1.1 Fri Oct 17 09:30:36 2003 --- dspam/md5.c Sat Nov 15 18:34:58 2003 *************** *** 48,58 **** #define S43 15 #define S44 21 ! static void MD5Transform PROTO_LIST ((UINT4 [4], unsigned char [64])); ! static void Encode PROTO_LIST ! ((unsigned char *, UINT4 *, unsigned int)); ! static void Decode PROTO_LIST ! ((UINT4 *, unsigned char *, unsigned int)); inline static void MD5_memcpy PROTO_LIST ((POINTER, POINTER, unsigned int)); inline static void MD5_memset PROTO_LIST ((POINTER, int, unsigned int)); --- 48,56 ---- #define S43 15 #define S44 21 ! static void MD5Transform PROTO_LIST ((UINT4[4], unsigned char[64])); ! static void Encode PROTO_LIST ((unsigned char *, UINT4 *, unsigned int)); ! static void Decode PROTO_LIST ((UINT4 *, unsigned char *, unsigned int)); inline static void MD5_memcpy PROTO_LIST ((POINTER, POINTER, unsigned int)); inline static void MD5_memset PROTO_LIST ((POINTER, int, unsigned int)); *************** *** 99,110 **** /* MD5 initialization. Begins an MD5 operation, writing a new context. */ ! void MD5Init (context) ! MD5_CTX *context; /* context */ { context->count[0] = context->count[1] = 0; /* Load magic initialization constants. ! */ context->state[0] = 0x67452301; context->state[1] = 0xefcdab89; context->state[2] = 0x98badcfe; --- 97,109 ---- /* MD5 initialization. Begins an MD5 operation, writing a new context. */ ! void ! MD5Init (context) ! MD5_CTX *context; /* context */ { context->count[0] = context->count[1] = 0; /* Load magic initialization constants. ! */ context->state[0] = 0x67452301; context->state[1] = 0xefcdab89; context->state[2] = 0x98badcfe; *************** *** 115,165 **** operation, processing another message block, and updating the context. */ ! void MD5Update (context, input, inputLen) ! MD5_CTX *context; /* context */ ! unsigned char *input; /* input block */ ! unsigned int inputLen; /* length of input block */ { unsigned int i, index, partLen; /* Compute number of bytes mod 64 */ ! index = (unsigned int)((context->count[0] >> 3) & 0x3F); /* Update number of bits */ ! if ((context->count[0] += ((UINT4)inputLen << 3)) ! < ((UINT4)inputLen << 3)) ! context->count[1]++; ! context->count[1] += ((UINT4)inputLen >> 29); partLen = 64 - index; /* Transform as many times as possible. ! */ ! if (inputLen >= partLen) { ! MD5_memcpy ! ((POINTER)&context->buffer[index], (POINTER)input, partLen); ! MD5Transform (context->state, context->buffer); ! for (i = partLen; i + 63 < inputLen; i += 64) ! MD5Transform (context->state, &input[i]); ! index = 0; } else ! i = 0; /* Buffer remaining input */ MD5_memcpy ! ((POINTER)&context->buffer[index], (POINTER)&input[i], ! inputLen-i); } /* MD5 finalization. Ends an MD5 message-digest operation, writing the the message digest and zeroizing the context. */ ! void MD5Final (digest, context) ! unsigned char digest[16]; /* message digest */ ! MD5_CTX *context; /* context */ { unsigned char bits[8]; unsigned int index, padLen; --- 114,165 ---- operation, processing another message block, and updating the context. */ ! void ! MD5Update (context, input, inputLen) ! MD5_CTX *context; /* context */ ! unsigned char *input; /* input block */ ! unsigned int inputLen; /* length of input block */ { unsigned int i, index, partLen; /* Compute number of bytes mod 64 */ ! index = (unsigned int) ((context->count[0] >> 3) & 0x3F); /* Update number of bits */ ! if ((context->count[0] += ((UINT4) inputLen << 3)) ! < ((UINT4) inputLen << 3)) ! context->count[1]++; ! context->count[1] += ((UINT4) inputLen >> 29); partLen = 64 - index; /* Transform as many times as possible. ! */ ! if (inputLen >= partLen) ! { ! MD5_memcpy ((POINTER) & context->buffer[index], (POINTER) input, partLen); ! MD5Transform (context->state, context->buffer); ! for (i = partLen; i + 63 < inputLen; i += 64) ! MD5Transform (context->state, &input[i]); ! index = 0; } else ! i = 0; /* Buffer remaining input */ MD5_memcpy ! ((POINTER) & context->buffer[index], (POINTER) & input[i], inputLen - i); } /* MD5 finalization. Ends an MD5 message-digest operation, writing the the message digest and zeroizing the context. */ ! void ! MD5Final (digest, context) ! unsigned char digest[16]; /* message digest */ ! MD5_CTX *context; /* context */ { unsigned char bits[8]; unsigned int index, padLen; *************** *** 168,175 **** Encode (bits, context->count, 8); /* Pad out to 56 mod 64. ! */ ! index = (unsigned int)((context->count[0] >> 3) & 0x3f); padLen = (index < 56) ? (56 - index) : (120 - index); MD5Update (context, PADDING, padLen); --- 168,175 ---- Encode (bits, context->count, 8); /* Pad out to 56 mod 64. ! */ ! index = (unsigned int) ((context->count[0] >> 3) & 0x3f); padLen = (index < 56) ? (56 - index) : (120 - index); MD5Update (context, PADDING, padLen); *************** *** 179,269 **** Encode (digest, context->state, 16); /* Zeroize sensitive information. ! */ ! MD5_memset ((POINTER)context, 0, sizeof (*context)); } /* MD5 basic transformation. Transforms state based on block. */ ! static void MD5Transform (state, block) ! UINT4 state[4]; ! unsigned char block[64]; { UINT4 a = state[0], b = state[1], c = state[2], d = state[3], x[16]; Decode (x, block, 64); /* Round 1 */ ! FF (a, b, c, d, x[ 0], S11, 0xd76aa478); /* 1 */ ! FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); /* 2 */ ! FF (c, d, a, b, x[ 2], S13, 0x242070db); /* 3 */ ! FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); /* 4 */ ! FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); /* 5 */ ! FF (d, a, b, c, x[ 5], S12, 0x4787c62a); /* 6 */ ! FF (c, d, a, b, x[ 6], S13, 0xa8304613); /* 7 */ ! FF (b, c, d, a, x[ 7], S14, 0xfd469501); /* 8 */ ! FF (a, b, c, d, x[ 8], S11, 0x698098d8); /* 9 */ ! FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); /* 10 */ ! FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */ ! FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */ ! FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */ ! FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */ ! FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */ ! FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */ ! ! /* Round 2 */ ! GG (a, b, c, d, x[ 1], S21, 0xf61e2562); /* 17 */ ! GG (d, a, b, c, x[ 6], S22, 0xc040b340); /* 18 */ ! GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */ ! GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); /* 20 */ ! GG (a, b, c, d, x[ 5], S21, 0xd62f105d); /* 21 */ ! GG (d, a, b, c, x[10], S22, 0x2441453); /* 22 */ ! GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */ ! GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); /* 24 */ ! GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); /* 25 */ ! GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */ ! GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); /* 27 */ ! GG (b, c, d, a, x[ 8], S24, 0x455a14ed); /* 28 */ ! GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */ ! GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); /* 30 */ ! GG (c, d, a, b, x[ 7], S23, 0x676f02d9); /* 31 */ ! GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */ /* Round 3 */ ! HH (a, b, c, d, x[ 5], S31, 0xfffa3942); /* 33 */ ! HH (d, a, b, c, x[ 8], S32, 0x8771f681); /* 34 */ ! HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */ ! HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */ ! HH (a, b, c, d, x[ 1], S31, 0xa4beea44); /* 37 */ ! HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); /* 38 */ ! HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); /* 39 */ ! HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */ ! HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */ ! HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); /* 42 */ ! HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); /* 43 */ ! HH (b, c, d, a, x[ 6], S34, 0x4881d05); /* 44 */ ! HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); /* 45 */ ! HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */ ! HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */ ! HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); /* 48 */ /* Round 4 */ ! II (a, b, c, d, x[ 0], S41, 0xf4292244); /* 49 */ ! II (d, a, b, c, x[ 7], S42, 0x432aff97); /* 50 */ ! II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */ ! II (b, c, d, a, x[ 5], S44, 0xfc93a039); /* 52 */ ! II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */ ! II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); /* 54 */ ! II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */ ! II (b, c, d, a, x[ 1], S44, 0x85845dd1); /* 56 */ ! II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); /* 57 */ ! II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */ ! II (c, d, a, b, x[ 6], S43, 0xa3014314); /* 59 */ ! II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */ ! II (a, b, c, d, x[ 4], S41, 0xf7537e82); /* 61 */ ! II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */ ! II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); /* 63 */ ! II (b, c, d, a, x[ 9], S44, 0xeb86d391); /* 64 */ state[0] += a; state[1] += b; --- 179,270 ---- Encode (digest, context->state, 16); /* Zeroize sensitive information. ! */ ! MD5_memset ((POINTER) context, 0, sizeof (*context)); } /* MD5 basic transformation. Transforms state based on block. */ ! static void ! MD5Transform (state, block) ! UINT4 state[4]; ! unsigned char block[64]; { UINT4 a = state[0], b = state[1], c = state[2], d = state[3], x[16]; Decode (x, block, 64); /* Round 1 */ ! FF (a, b, c, d, x[0], S11, 0xd76aa478); /* 1 */ ! FF (d, a, b, c, x[1], S12, 0xe8c7b756); /* 2 */ ! FF (c, d, a, b, x[2], S13, 0x242070db); /* 3 */ ! FF (b, c, d, a, x[3], S14, 0xc1bdceee); /* 4 */ ! FF (a, b, c, d, x[4], S11, 0xf57c0faf); /* 5 */ ! FF (d, a, b, c, x[5], S12, 0x4787c62a); /* 6 */ ! FF (c, d, a, b, x[6], S13, 0xa8304613); /* 7 */ ! FF (b, c, d, a, x[7], S14, 0xfd469501); /* 8 */ ! FF (a, b, c, d, x[8], S11, 0x698098d8); /* 9 */ ! FF (d, a, b, c, x[9], S12, 0x8b44f7af); /* 10 */ ! FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */ ! FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */ ! FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */ ! FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */ ! FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */ ! FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */ ! ! /* Round 2 */ ! GG (a, b, c, d, x[1], S21, 0xf61e2562); /* 17 */ ! GG (d, a, b, c, x[6], S22, 0xc040b340); /* 18 */ ! GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */ ! GG (b, c, d, a, x[0], S24, 0xe9b6c7aa); /* 20 */ ! GG (a, b, c, d, x[5], S21, 0xd62f105d); /* 21 */ ! GG (d, a, b, c, x[10], S22, 0x2441453); /* 22 */ ! GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */ ! GG (b, c, d, a, x[4], S24, 0xe7d3fbc8); /* 24 */ ! GG (a, b, c, d, x[9], S21, 0x21e1cde6); /* 25 */ ! GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */ ! GG (c, d, a, b, x[3], S23, 0xf4d50d87); /* 27 */ ! GG (b, c, d, a, x[8], S24, 0x455a14ed); /* 28 */ ! GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */ ! GG (d, a, b, c, x[2], S22, 0xfcefa3f8); /* 30 */ ! GG (c, d, a, b, x[7], S23, 0x676f02d9); /* 31 */ ! GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */ /* Round 3 */ ! HH (a, b, c, d, x[5], S31, 0xfffa3942); /* 33 */ ! HH (d, a, b, c, x[8], S32, 0x8771f681); /* 34 */ ! HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */ ! HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */ ! HH (a, b, c, d, x[1], S31, 0xa4beea44); /* 37 */ ! HH (d, a, b, c, x[4], S32, 0x4bdecfa9); /* 38 */ ! HH (c, d, a, b, x[7], S33, 0xf6bb4b60); /* 39 */ ! HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */ ! HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */ ! HH (d, a, b, c, x[0], S32, 0xeaa127fa); /* 42 */ ! HH (c, d, a, b, x[3], S33, 0xd4ef3085); /* 43 */ ! HH (b, c, d, a, x[6], S34, 0x4881d05); /* 44 */ ! HH (a, b, c, d, x[9], S31, 0xd9d4d039); /* 45 */ ! HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */ ! HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */ ! HH (b, c, d, a, x[2], S34, 0xc4ac5665); /* 48 */ /* Round 4 */ ! II (a, b, c, d, x[0], S41, 0xf4292244); /* 49 */ ! II (d, a, b, c, x[7], S42, 0x432aff97); /* 50 */ ! II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */ ! II (b, c, d, a, x[5], S44, 0xfc93a039); /* 52 */ ! II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */ ! II (d, a, b, c, x[3], S42, 0x8f0ccc92); /* 54 */ ! II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */ ! II (b, c, d, a, x[1], S44, 0x85845dd1); /* 56 */ ! II (a, b, c, d, x[8], S41, 0x6fa87e4f); /* 57 */ ! II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */ ! II (c, d, a, b, x[6], S43, 0xa3014314); /* 59 */ ! II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */ ! II (a, b, c, d, x[4], S41, 0xf7537e82); /* 61 */ ! II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */ ! II (c, d, a, b, x[2], S43, 0x2ad7d2bb); /* 63 */ ! II (b, c, d, a, x[9], S44, 0xeb86d391); /* 64 */ state[0] += a; state[1] += b; *************** *** 271,323 **** state[3] += d; /* Zeroize sensitive information. ! */ ! MD5_memset ((POINTER)x, 0, sizeof (x)); } /* Encodes input (UINT4) into output (unsigned char). Assumes len is a multiple of 4. */ ! static void Encode (output, input, len) ! unsigned char *output; ! UINT4 *input; ! unsigned int len; { unsigned int i, j; ! for (i = 0, j = 0; j < len; i++, j += 4) { ! output[j] = (unsigned char)(input[i] & 0xff); ! output[j+1] = (unsigned char)((input[i] >> 8) & 0xff); ! output[j+2] = (unsigned char)((input[i] >> 16) & 0xff); ! output[j+3] = (unsigned char)((input[i] >> 24) & 0xff); } } /* Decodes input (unsigned char) into output (UINT4). Assumes len is a multiple of 4. */ ! static void Decode (output, input, len) ! UINT4 *output; ! unsigned char *input; ! unsigned int len; { unsigned int i, j; for (i = 0, j = 0; j < len; i++, j += 4) ! output[i] = ((UINT4)input[j]) | (((UINT4)input[j+1]) << 8) | ! (((UINT4)input[j+2]) << 16) | (((UINT4)input[j+3]) << 24); } /* Note: Replace "for loop" with standard memcpy if possible. */ ! static void MD5_memcpy (output, input, len) ! POINTER output; ! POINTER input; ! unsigned int len; { #ifdef HAVE_MEMCPY ! memcpy(output, input, len); #else unsigned int i; for (i = 0; i < len; i++) --- 272,328 ---- state[3] += d; /* Zeroize sensitive information. ! */ ! MD5_memset ((POINTER) x, 0, sizeof (x)); } /* Encodes input (UINT4) into output (unsigned char). Assumes len is a multiple of 4. */ ! static void ! Encode (output, input, len) ! unsigned char *output; ! UINT4 *input; ! unsigned int len; { unsigned int i, j; ! for (i = 0, j = 0; j < len; i++, j += 4) ! { ! output[j] = (unsigned char) (input[i] & 0xff); ! output[j + 1] = (unsigned char) ((input[i] >> 8) & 0xff); ! output[j + 2] = (unsigned char) ((input[i] >> 16) & 0xff); ! output[j + 3] = (unsigned char) ((input[i] >> 24) & 0xff); } } /* Decodes input (unsigned char) into output (UINT4). Assumes len is a multiple of 4. */ ! static void ! Decode (output, input, len) ! UINT4 *output; ! unsigned char *input; ! unsigned int len; { unsigned int i, j; for (i = 0, j = 0; j < len; i++, j += 4) ! output[i] = ((UINT4) input[j]) | (((UINT4) input[j + 1]) << 8) | ! (((UINT4) input[j + 2]) << 16) | (((UINT4) input[j + 3]) << 24); } /* Note: Replace "for loop" with standard memcpy if possible. */ ! static void ! MD5_memcpy (output, input, len) ! POINTER output; ! POINTER input; ! unsigned int len; { #ifdef HAVE_MEMCPY ! memcpy (output, input, len); #else unsigned int i; for (i = 0; i < len; i++) *************** *** 327,342 **** /* Note: Replace "for loop" with standard memset if possible. */ ! static void MD5_memset (output, value, len) ! POINTER output; ! int value; ! unsigned int len; { #ifdef HAVE_MEMSET ! memset(output, value, len); #else unsigned int i; for (i = 0; i < len; i++) ! ((char *)output)[i] = (char)value; #endif } --- 332,348 ---- /* Note: Replace "for loop" with standard memset if possible. */ ! static void ! MD5_memset (output, value, len) ! POINTER output; ! int value; ! unsigned int len; { #ifdef HAVE_MEMSET ! memset (output, value, len); #else unsigned int i; for (i = 0; i < len; i++) ! ((char *) output)[i] = (char) value; #endif } Index: dspam/mysql_drv.c diff -c dspam/mysql_drv.c:1.1.1.1 dspam/mysql_drv.c:1.1.1.1.2.1 *** dspam/mysql_drv.c:1.1.1.1 Thu Oct 30 11:22:07 2003 --- dspam/mysql_drv.c Sat Nov 15 18:34:58 2003 *************** *** 60,66 **** { return 0; } ! int dspam_shutdown_driver (void) { --- 60,66 ---- { return 0; } ! int dspam_shutdown_driver (void) { *************** *** 82,88 **** return EINVAL; } ! if (! CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); --- 82,88 ---- return EINVAL; } ! if (!CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); *************** *** 119,125 **** CTX->totals.spam_misclassified = strtol (row[2], NULL, 0); CTX->totals.innocent_misclassified = strtol (row[3], NULL, 0); CTX->totals.spam_corpusfed = strtol (row[4], NULL, 0); ! CTX->totals.innocent_corpusfed = strtol(row[5], NULL, 0); mysql_free_result (result); --- 119,125 ---- CTX->totals.spam_misclassified = strtol (row[2], NULL, 0); CTX->totals.innocent_misclassified = strtol (row[3], NULL, 0); CTX->totals.spam_corpusfed = strtol (row[4], NULL, 0); ! CTX->totals.innocent_corpusfed = strtol (row[5], NULL, 0); mysql_free_result (result); *************** *** 142,152 **** if (CTX->mode == DSM_CLASSIFY) { ! _mysql_drv_get_spamtotals (CTX); /* undo changes to in memory totals */ return 0; } ! if (! CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); --- 142,152 ---- if (CTX->mode == DSM_CLASSIFY) { ! _mysql_drv_get_spamtotals (CTX); /* undo changes to in memory totals */ return 0; } ! if (!CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); *************** *** 166,175 **** CTX->totals.total_innocent, CTX->totals.spam_misclassified, CTX->totals.innocent_misclassified, CTX->totals.spam_corpusfed, CTX->totals.innocent_corpusfed); ! result = mysql_query(s->dbh, query); } ! if (s->control_totals.total_innocent != 0 || result) { snprintf (query, sizeof (query), "update dspam_stats set total_spam = total_spam %s %d, " --- 166,175 ---- CTX->totals.total_innocent, CTX->totals.spam_misclassified, CTX->totals.innocent_misclassified, CTX->totals.spam_corpusfed, CTX->totals.innocent_corpusfed); ! result = mysql_query (s->dbh, query); } ! if (s->control_totals.total_innocent != 0 || result) { snprintf (query, sizeof (query), "update dspam_stats set total_spam = total_spam %s %d, " *************** *** 234,240 **** return EINVAL; } ! if (! CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); --- 234,240 ---- return EINVAL; } ! if (!CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); *************** *** 299,305 **** stat.disk = 'Y'; #ifdef VERBOSE ! _mysql_drv_query_error ( "token returned", row[0]); #endif /* our control token will tell us how each token was altered */ --- 299,305 ---- stat.disk = 'Y'; #ifdef VERBOSE ! _mysql_drv_query_error ("token returned", row[0]); #endif /* our control token will tell us how each token was altered */ *************** *** 345,351 **** if (CTX->mode == DSM_CLASSIFY) return 0; ! if (! CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); --- 345,351 ---- if (CTX->mode == DSM_CLASSIFY) return 0; ! if (!CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); *************** *** 415,421 **** { /* possible that it got inserted in-between dspam agent runs? */ _mysql_drv_query_error (mysql_error (s->dbh), insert); ! stat.disk = 'Y'; } } --- 415,421 ---- { /* possible that it got inserted in-between dspam agent runs? */ _mysql_drv_query_error (mysql_error (s->dbh), insert); ! stat.disk = 'Y'; } } *************** *** 462,468 **** return EINVAL; } ! if (! CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); --- 462,468 ---- return EINVAL; } ! if (!CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); *************** *** 525,531 **** if (CTX->mode == DSM_CLASSIFY) return 0; ! if (! CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); --- 525,531 ---- if (CTX->mode == DSM_CLASSIFY) return 0; ! if (!CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); *************** *** 642,653 **** return EUNKNOWN; } ! if (hostname[0] == '/') { connect = mysql_real_connect (s->dbh, NULL, user, password, db, 0, hostname, 0); ! } else { connect = ! mysql_real_connect (s->dbh, hostname, user, password, db, port, NULL, 0); } if (connect == NULL) { --- 642,657 ---- return EUNKNOWN; } ! if (hostname[0] == '/') ! { connect = mysql_real_connect (s->dbh, NULL, user, password, db, 0, hostname, 0); ! } ! else ! { connect = ! mysql_real_connect (s->dbh, hostname, user, password, db, port, NULL, ! 0); } if (connect == NULL) { *************** *** 742,748 **** return EINVAL; } ! if (! CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); --- 746,752 ---- return EINVAL; } ! if (!CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); *************** *** 815,821 **** return EINVAL; } ! if (! CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); --- 819,825 ---- return EINVAL; } ! if (!CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); *************** *** 872,878 **** return EINVAL; } ! if (! CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); --- 876,882 ---- return EINVAL; } ! if (!CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); *************** *** 911,917 **** return EINVAL; } ! if (! CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); --- 915,921 ---- return EINVAL; } ! if (!CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); *************** *** 968,974 **** if (s->iter_user == NULL) { #ifdef VIRTUAL_USERS ! strcpy(query, "select distinct username from dspam_virtual_uids"); #else strcpy (query, "select distinct uid from dspam_stats"); #endif --- 972,978 ---- if (s->iter_user == NULL) { #ifdef VIRTUAL_USERS ! strcpy (query, "select distinct username from dspam_virtual_uids"); #else strcpy (query, "select distinct uid from dspam_stats"); #endif *************** *** 992,998 **** } #ifdef VIRTUAL_USERS ! strlcpy(user, row[0], sizeof(user)); #else uid = (uid_t) atoi (row[0]); p = _mysql_drv_getpwuid (CTX, uid); --- 996,1002 ---- } #ifdef VIRTUAL_USERS ! strlcpy (user, row[0], sizeof (user)); #else uid = (uid_t) atoi (row[0]); p = _mysql_drv_getpwuid (CTX, uid); *************** *** 1024,1030 **** return NULL; } ! if (! CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); --- 1028,1034 ---- return NULL; } ! if (!CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); *************** *** 1094,1100 **** return NULL; } ! if (! CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); --- 1098,1104 ---- return NULL; } ! if (!CTX->group) p = _mysql_drv_getpwnam (CTX, CTX->username); else p = _mysql_drv_getpwnam (CTX, CTX->group); *************** *** 1173,1182 **** if (p.pw_name != NULL) { /* cache the last name queried */ ! if (name != NULL && !strcmp (p.pw_name, name)) return &p; ! ! free(p.pw_name); p.pw_name = NULL; p.pw_uid = 0; } --- 1177,1186 ---- if (p.pw_name != NULL) { /* cache the last name queried */ ! if (name != NULL && !strcmp (p.pw_name, name)) return &p; ! ! free (p.pw_name); p.pw_name = NULL; p.pw_uid = 0; } *************** *** 1185,1191 **** if (q == NULL) return NULL; p.pw_uid = q->pw_uid; ! p.pw_name = strdup(q->pw_name); return &p; #else --- 1189,1195 ---- if (q == NULL) return NULL; p.pw_uid = q->pw_uid; ! p.pw_name = strdup (q->pw_name); return &p; #else *************** *** 1247,1263 **** static struct passwd p = { NULL, NULL, 0, 0, NULL, NULL, NULL }; struct passwd *q; ! if (p.pw_name != NULL) { /* cache the last uid queried */ ! if (p.pw_uid == uid) { return &p; } p.pw_name = NULL; } q = getpwuid (uid); ! memcpy(&p, q, sizeof(struct passwd)); return &p; #else --- 1251,1268 ---- static struct passwd p = { NULL, NULL, 0, 0, NULL, NULL, NULL }; struct passwd *q; ! if (p.pw_name != NULL) { /* cache the last uid queried */ ! if (p.pw_uid == uid) ! { return &p; } p.pw_name = NULL; } q = getpwuid (uid); ! memcpy (&p, q, sizeof (struct passwd)); return &p; #else *************** *** 1318,1326 **** char ct[128]; char fn[MAX_FILENAME_LENGTH]; ! LOG (LOG_WARNING, "query error: %s: see sql.errors for more details", error); ! snprintf(fn, sizeof(fn), "%s/sql.errors", USERDIR); snprintf (ct, sizeof (ct), "%s", ctime (&tm)); chomp (ct); --- 1323,1332 ---- char ct[128]; char fn[MAX_FILENAME_LENGTH]; ! LOG (LOG_WARNING, "query error: %s: see sql.errors for more details", ! error); ! snprintf (fn, sizeof (fn), "%s/sql.errors", USERDIR); snprintf (ct, sizeof (ct), "%s", ctime (&tm)); chomp (ct); *************** *** 1350,1356 **** name); /* we need to fail, to prevent a potential loop - even if it was inserted ! by another process */ if (mysql_query (s->dbh, query)) { _mysql_drv_query_error (mysql_error (s->dbh), query); --- 1356,1362 ---- name); /* we need to fail, to prevent a potential loop - even if it was inserted ! * by another process */ if (mysql_query (s->dbh, query)) { _mysql_drv_query_error (mysql_error (s->dbh), query); Index: dspam/ora_drv.c diff -c dspam/ora_drv.c:1.1.1.1 dspam/ora_drv.c:1.1.1.1.2.1 *** dspam/ora_drv.c:1.1.1.1 Thu Oct 30 17:14:14 2003 --- dspam/ora_drv.c Sat Nov 15 18:34:58 2003 *************** *** 60,66 **** { return 0; } ! int dspam_shutdown_driver (void) { --- 60,66 ---- { return 0; } ! int dspam_shutdown_driver (void) { *************** *** 246,252 **** char ct[128]; char fn[MAX_FILENAME_LENGTH]; ! snprintf(fn, sizeof(fn), "%s/sql.errors", USERDIR); snprintf (ct, sizeof (ct), "%s", ctime (&tm)); chomp (ct); --- 246,252 ---- char ct[128]; char fn[MAX_FILENAME_LENGTH]; ! snprintf (fn, sizeof (fn), "%s/sql.errors", USERDIR); snprintf (ct, sizeof (ct), "%s", ctime (&tm)); chomp (ct); *************** *** 1106,1129 **** #ifndef VIRTUAL_USERS static struct passwd p = { NULL, NULL, 0, 0, NULL, NULL, NULL }; struct passwd *q; ! if (p.pw_name != NULL) { /* cache the last name queried */ if (name != NULL && !strcmp (p.pw_name, name)) return &p; ! ! free(p.pw_name); p.pw_name = NULL; p.pw_uid = 0; } ! q = getpwnam (name); if (q == NULL) return NULL; p.pw_uid = q->pw_uid; ! p.pw_name = strdup(q->pw_name); ! return &p; #else struct _ora_drv_storage *s = (struct _ora_drv_storage *) CTX->storage; --- 1106,1129 ---- #ifndef VIRTUAL_USERS static struct passwd p = { NULL, NULL, 0, 0, NULL, NULL, NULL }; struct passwd *q; ! if (p.pw_name != NULL) { /* cache the last name queried */ if (name != NULL && !strcmp (p.pw_name, name)) return &p; ! ! free (p.pw_name); p.pw_name = NULL; p.pw_uid = 0; } ! q = getpwnam (name); if (q == NULL) return NULL; p.pw_uid = q->pw_uid; ! p.pw_name = strdup (q->pw_name); ! return &p; #else struct _ora_drv_storage *s = (struct _ora_drv_storage *) CTX->storage; *************** *** 1207,1225 **** #ifndef VIRTUAL_USERS static struct passwd p = { NULL, NULL, 0, 0, NULL, NULL, NULL }; struct passwd *q; ! ! if (p.pw_name != NULL) { /* cache the last uid queried */ ! if (p.pw_uid == uid) { return &p; } p.pw_name = NULL; } ! q = getpwuid (uid); ! memcpy(&p, q, sizeof(struct passwd)); ! return &p; #else struct _ora_drv_storage *s = (struct _ora_drv_storage *) CTX->storage; --- 1207,1226 ---- #ifndef VIRTUAL_USERS static struct passwd p = { NULL, NULL, 0, 0, NULL, NULL, NULL }; struct passwd *q; ! ! if (p.pw_name != NULL) { /* cache the last uid queried */ ! if (p.pw_uid == uid) ! { return &p; } p.pw_name = NULL; } ! q = getpwuid (uid); ! memcpy (&p, q, sizeof (struct passwd)); ! return &p; #else struct _ora_drv_storage *s = (struct _ora_drv_storage *) CTX->storage; *************** *** 1510,1517 **** if (CTX->mode == DSM_CLASSIFY) { ! _ora_drv_get_spamtotals (CTX); /* undo changes to in memory totals */ ! return 0; } p = _ora_drv_getpwnam (CTX, CTX->username); --- 1511,1518 ---- if (CTX->mode == DSM_CLASSIFY) { ! _ora_drv_get_spamtotals (CTX); /* undo changes to in memory totals */ ! return 0; } p = _ora_drv_getpwnam (CTX, CTX->username); *************** *** 1531,1567 **** s->schema, p->pw_uid, CTX->totals.total_spam, CTX->totals.total_innocent, CTX->totals.spam_misclassified, ! CTX->totals.innocent_misclassified, ! CTX->totals.spam_corpusfed, ! CTX->totals.innocent_corpusfed); if (_ora_drv_checkerr (NULL, s->errhp, OCIHandleAlloc ((dvoid *) s->envhp, ! (dvoid **) & stmthp, ! (ub4) OCI_HTYPE_STMT, ! (size_t) 0, ! (dvoid **) 0)) != ! OCI_SUCCESS) return EUNKNOWN; ! if (_ora_drv_checkerr (query, s->errhp, OCIStmtPrepare (stmthp, s->errhp, ! (text *) query, ! (ub4) ! strlen (query), ! (ub4) ! OCI_NTV_SYNTAX, ! (ub4) OCI_DEFAULT)) ! != OCI_SUCCESS) goto bail; ! if (_ora_drv_checkerr (query, s->errhp, OCIStmtExecute (s->svchp, stmthp, ! s->errhp, (ub4) 1, ! (ub4) 0, ! (OCISnapshot *) ! NULL, ! (OCISnapshot *) ! NULL, ! (ub4) OCI_DEFAULT)) ! != OCI_SUCCESS) bad = 1; } --- 1532,1571 ---- s->schema, p->pw_uid, CTX->totals.total_spam, CTX->totals.total_innocent, CTX->totals.spam_misclassified, ! CTX->totals.innocent_misclassified, ! CTX->totals.spam_corpusfed, CTX->totals.innocent_corpusfed); if (_ora_drv_checkerr (NULL, s->errhp, OCIHandleAlloc ((dvoid *) s->envhp, ! (dvoid **) & ! stmthp, ! (ub4) ! OCI_HTYPE_STMT, ! (size_t) 0, ! (dvoid **) 0)) != ! OCI_SUCCESS) return EUNKNOWN; ! if (_ora_drv_checkerr (query, s->errhp, OCIStmtPrepare (stmthp, s->errhp, ! (text *) query, ! (ub4) ! strlen (query), ! (ub4) ! OCI_NTV_SYNTAX, ! (ub4) ! OCI_DEFAULT)) != ! OCI_SUCCESS) goto bail; ! if (_ora_drv_checkerr (query, s->errhp, OCIStmtExecute (s->svchp, stmthp, ! s->errhp, (ub4) 1, ! (ub4) 0, ! (OCISnapshot *) ! NULL, ! (OCISnapshot *) ! NULL, ! (ub4) ! OCI_DEFAULT)) != ! OCI_SUCCESS) bad = 1; } *************** *** 1601,1632 **** s->control_totals.innocent_corpusfed), p->pw_uid); if (_ora_drv_checkerr (NULL, s->errhp, OCIHandleAlloc ((dvoid *) s->envhp, ! (dvoid **) & stmthp, ! (ub4) OCI_HTYPE_STMT, ! (size_t) 0, ! (dvoid **) 0)) != ! OCI_SUCCESS) return EUNKNOWN; if (_ora_drv_checkerr (query, s->errhp, OCIStmtPrepare (stmthp, s->errhp, ! (text *) query, ! (ub4) ! strlen (query), ! (ub4) ! OCI_NTV_SYNTAX, ! (ub4) OCI_DEFAULT)) ! != OCI_SUCCESS) goto bail; if (_ora_drv_checkerr (query, s->errhp, OCIStmtExecute (s->svchp, stmthp, ! s->errhp, (ub4) 1, ! (ub4) 0, ! (OCISnapshot *) ! NULL, ! (OCISnapshot *) ! NULL, ! (ub4) OCI_DEFAULT)) ! != OCI_SUCCESS) goto bail; } --- 1605,1640 ---- s->control_totals.innocent_corpusfed), p->pw_uid); if (_ora_drv_checkerr (NULL, s->errhp, OCIHandleAlloc ((dvoid *) s->envhp, ! (dvoid **) & ! stmthp, ! (ub4) ! OCI_HTYPE_STMT, ! (size_t) 0, ! (dvoid **) 0)) != ! OCI_SUCCESS) return EUNKNOWN; if (_ora_drv_checkerr (query, s->errhp, OCIStmtPrepare (stmthp, s->errhp, ! (text *) query, ! (ub4) ! strlen (query), ! (ub4) ! OCI_NTV_SYNTAX, ! (ub4) ! OCI_DEFAULT)) != ! OCI_SUCCESS) goto bail; if (_ora_drv_checkerr (query, s->errhp, OCIStmtExecute (s->svchp, stmthp, ! s->errhp, (ub4) 1, ! (ub4) 0, ! (OCISnapshot *) ! NULL, ! (OCISnapshot *) ! NULL, ! (ub4) ! OCI_DEFAULT)) != ! OCI_SUCCESS) goto bail; } *************** *** 1894,1900 **** OCIStmtExecute (s->svchp, stmthp, s->errhp, (ub4) 1, (ub4) 0, (OCISnapshot *) NULL, (OCISnapshot *) NULL, (ub4) OCI_DEFAULT)) != OCI_SUCCESS) ! stat.disk = 'Y'; } if (stat.disk == 'Y') --- 1902,1908 ---- OCIStmtExecute (s->svchp, stmthp, s->errhp, (ub4) 1, (ub4) 0, (OCISnapshot *) NULL, (OCISnapshot *) NULL, (ub4) OCI_DEFAULT)) != OCI_SUCCESS) ! stat.disk = 'Y'; } if (stat.disk == 'Y') Index: dspam/storage_driver.h diff -c dspam/storage_driver.h:1.1.1.1 dspam/storage_driver.h:1.1.1.1.2.1 *** dspam/storage_driver.h:1.1.1.1 Sun Nov 2 14:55:23 2003 --- dspam/storage_driver.h Sat Nov 15 18:34:58 2003 *************** *** 74,81 **** struct _ds_spam_stat *stat); /* Iteration calls */ ! struct _ds_storage_record *_ds_get_nexttoken (DSPAM_CTX * CTX); ! struct _ds_storage_signature *_ds_get_nextsignature (DSPAM_CTX * CTX); char *_ds_get_nextuser (DSPAM_CTX * CTX); /* Signature processing calls */ --- 74,81 ---- struct _ds_spam_stat *stat); /* Iteration calls */ ! struct _ds_storage_record *_ds_get_nexttoken (DSPAM_CTX * CTX); ! struct _ds_storage_signature *_ds_get_nextsignature (DSPAM_CTX * CTX); char *_ds_get_nextuser (DSPAM_CTX * CTX); /* Signature processing calls */ Index: dspam/tbt.c diff -c dspam/tbt.c:1.1.1.3 dspam/tbt.c:1.1.1.3.2.1 *** dspam/tbt.c:1.1.1.3 Fri Oct 24 09:32:12 2003 --- dspam/tbt.c Sat Nov 15 18:34:58 2003 *************** *** 134,140 **** } int ! tbt_add (struct tbt *tbt, double probability, unsigned long long token, unsigned long frequency) { struct tbt_node *node; struct tbt_node *search, *parent; --- 134,141 ---- } int ! tbt_add (struct tbt *tbt, double probability, unsigned long long token, ! unsigned long frequency) { struct tbt_node *node; struct tbt_node *search, *parent; *************** *** 158,164 **** } delta = fabs (.5 - probability); ! parent = tbt->root; search = parent; --- 159,165 ---- } delta = fabs (.5 - probability); ! parent = tbt->root; search = parent; *************** *** 171,181 **** { d = right; search = parent->right; ! } else { ! if (frequency > search->frequency) { d = left; search = parent->left; ! } else { d = right; search = parent->right; } --- 172,187 ---- { d = right; search = parent->right; ! } ! else ! { ! if (frequency > search->frequency) ! { d = left; search = parent->left; ! } ! else ! { d = right; search = parent->right; } *************** *** 193,203 **** { d = right; search = parent->right; ! } else { ! if (frequency > search->frequency) { d = left; search = parent->left; ! } else { d = right; search = parent->right; } --- 199,214 ---- { d = right; search = parent->right; ! } ! else ! { ! if (frequency > search->frequency) ! { d = left; search = parent->left; ! } ! else ! { d = right; search = parent->right; } Index: dspam/testlibdspam.c diff -c /dev/null dspam/testlibdspam.c:1.24.2.2 *** /dev/null Sat Nov 15 20:45:12 2003 --- dspam/testlibdspam.c Sat Nov 15 19:58:36 2003 *************** *** 0 **** --- 1,608 ---- + #include + #include "libdspam.h" + #include "libdspam_objects.h" + #include "tbt.h" + #include + + #ifdef _AIX + #undef RAND_MAX /* AIX defines incorrect value for RAND_MAX */ + #define RAND_MAX 2147483647 + #endif + + #define DSPAM_API 28 + + #if DSPAM_API < 28 /* 2.6 API */ + const char *fname = "/tmp/test.dict"; + + static int compare_sig( + struct _ds_spam_signature *a, + struct _ds_spam_signature *b) { + return (a->length == b->length) ? memcmp(a->data,b->data,a->length) : 1; + } + + #else /* 2.8 API */ + #define fname "testuser",0 + #define spam_misses spam_misclassified + #define false_positives innocent_misclassified + extern void dspam_init_driver(); + extern void dspam_shutdown_driver(); + + static void resetuser(const char *user,const char *group) { + char cmd[80]; + DSPAM_CTX *ctx; + sprintf(cmd,"rm -rf /tmp/%s",user); + system(cmd); + ctx = dspam_init(user,group,DSM_PROCESS,DSF_CHAINED); + dspam_destroy(ctx); + } + + static int compare_sig( + struct _ds_spam_signature *a, + struct _ds_spam_signature *b) { + struct _ds_signature_token *p = a->data, *q = b->data; + int plen = a->length / sizeof *p; + int qlen = b->length / sizeof *q; + int i; + if (plen != qlen) return 1; + for (i = 0; i < plen; ++i) { + if (p[i].token != q[i].token || p[i].frequency != q[i].frequency) + return 1; + } + return 0; + } + #endif + + static const char msg1[] = "\ + From user@domain.com\n\ + Subject: Test message\n\ + To: testsys\n\ + \n\ + Testing 1 2 3\n\ + "; + + static const char spam1[] = "\ + From jerk@parasite.slime\n\ + Subject: RE: Info you requested\n\ + To: victim@lamb.com\n\ + \n\ + Limited time offer!\n\ + Click here to unsubscribe\n\ + "; + + static int + _dspam_process(DSPAM_CTX *ctx,const char *msg, int r, + const char *file,int line) { + int rc; + _fail_unless(ctx != 0,file,line,"init context failed"); + if (!ctx) return -1; + rc = dspam_process(ctx,msg); + if (ctx->message) { + _ds_destroy_message(ctx->message); + ctx->message = 0; + } + if (rc != r) { + char buf[80]; + sprintf(buf,"dspam_process returned %d, expected %d",rc,r); + _fail_unless(rc == r,file,line,buf); + } + return rc; + } + + #define dspam_process(ctx,msg) _dspam_process(ctx,msg,0,__FILE__,__LINE__) + #define dspam_process_rc(ctx,msg,rc) \ + _dspam_process(ctx,msg,rc,__FILE__,__LINE__) + + /* Check intended usage of CORPUS option. */ + START_TEST(test_corpus) { + DSPAM_CTX *ctx; + resetuser(fname); + ctx = dspam_init(fname,DSM_PROCESS,DSF_CHAINED|DSF_CORPUS); + dspam_process(ctx,msg1); + fail_unless(ctx->result == DSR_ISINNOCENT,"result not INNOCENT"); + fail_unless(ctx->totals.total_spam == 0,"total spam not 0"); + fail_unless(ctx->totals.total_innocent == 1,"total innocent not 1"); + dspam_destroy(ctx); + ctx = dspam_init(fname,DSM_ADDSPAM,DSF_CHAINED|DSF_CORPUS); + dspam_process(ctx,spam1); + fail_unless(ctx->result == DSR_ISSPAM,"result not SPAM"); + fail_unless(ctx->totals.total_spam == 1,"total spam not 1"); + fail_unless(ctx->totals.total_innocent == 1,"total innocent not 1"); + #if DSPAM_API == 28 + fail_unless(ctx->totals.spam_corpusfed == 1,"total corpus spam not 1"); + #else + /* beginning with 2.6.4, DSF_ADDSPAM+DSF_CORPUS counts as a miss */ + fail_unless(ctx->totals.spam_misses == 1,"total misses not 1"); + #endif + fail_unless(ctx->totals.false_positives == 0,"total fp not 0"); + /* ramp spam stats until spam1 is recognized as such */ + { int i; + for (i = 0; i < 20; ++i) + dspam_process(ctx,spam1); + dspam_destroy(ctx); + ctx = dspam_init(fname,DSM_PROCESS,DSF_CHAINED|DSF_CORPUS); + for (i = 0; i < 20; ++i) + dspam_process(ctx,msg1); + } + dspam_destroy(ctx); + ctx = dspam_init(fname,DSM_CLASSIFY,DSF_CHAINED); + dspam_process(ctx,spam1); + fail_unless(ctx->result == DSR_ISSPAM,"result not SPAM"); + dspam_destroy(ctx); + ctx = dspam_init(fname,DSM_PROCESS,DSF_CHAINED|DSF_CORPUS); + dspam_process(ctx,spam1); + fail_unless(ctx->result == DSR_ISINNOCENT,"result not INNOCENT"); + dspam_destroy(ctx); + } END_TEST + + static const char nasty1[] = "\ + From jerk@parasite.slime\n\ + Subject: RE: Info you requested\n\ + To: victim@lamb.com\n\ + This-Is-A-Really-Big-Header-That-Is-Designed-To-See-Whether-The-Fixed-Size\ + -Heading-Buffer-Causes-Any-Problems-With-Overflow-And-Possibly-Executing\ + -Arbitrary-Code: You Lose Sucker\n\ + \n\ + Bwa! Ha! Ha! Ha! Thisisareallylongtokenthatislongerthan25chars.\n\ + Click here to unsubscribe\n\ + "; + static const char nasty2[] = "\ + From: \"Farica Anderson\" \n\ + To: victim@lamb.com\n\ + Subject: Download this!\n\ + Date: Wed, 09 Jul 2003 15:57:36 +0000\n\ + MIME-Version: 1.0\n\ + Content-Type: text/html\n\ + Content-Transfer-Encoding: 8bit\n\ + \n\ + \n\ + "; + + /** Check possible overflow situations. Mostly, dspam checks for and ignores + * extra chars on long headings and tokens, but we check to make sure the + * checking still works. */ + START_TEST(test_overflow) { + DSPAM_CTX *ctx; + resetuser(fname); + ctx = dspam_init(fname,DSM_PROCESS,DSF_CHAINED|DSF_CORPUS); + dspam_process(ctx,nasty1); + dspam_destroy(ctx); + /* This little bugger crashes 2.6.2. */ + ctx = dspam_init(fname,DSM_ADDSPAM,DSF_CHAINED|DSF_IGNOREHEADER); + dspam_process_rc(ctx,nasty2,-2); /* -2 returned when no tokens found */ + dspam_destroy(ctx); + } END_TEST + + /* Check intended usage of CLASSIFY option. No updates should take + * place. Should be able to add signature result later with CORPUS option. */ + START_TEST(test_classify) { + struct _ds_spam_totals tot; + struct _ds_spam_signature sig1,sig2; /* signature objects */ + DSPAM_CTX *ctx; + resetuser(fname); + ctx = dspam_init(fname,DSM_CLASSIFY, DSF_CHAINED|DSF_SIGNATURE); + dspam_process(ctx,msg1); + fail_unless(ctx->result == DSR_ISINNOCENT,"result not INNOCENT"); + fail_unless(ctx->result > 0,"dspam result not positive"); + tot = ctx->totals; + sig1 = *ctx->signature; ctx->signature->data = NULL; + dspam_destroy(ctx); + ctx = dspam_init(fname,DSM_CLASSIFY, DSF_CHAINED|DSF_SIGNATURE); + dspam_process(ctx,spam1); + /* check that on_disk totals didn't change with classify */ + fail_unless(ctx->totals.total_innocent == tot.total_innocent, + "disk totals changed with CLASSIFY"); + /* check that in memory totals didn't change with classify */ + fail_unless(tot.total_innocent == 0,"memory stats changed with CLASSIFY"); + sig2 = *ctx->signature; ctx->signature->data = NULL; + dspam_destroy(ctx); + /* test updating with signature after CLASSIFY */ + ctx = dspam_init(fname,DSM_ADDSPAM,DSF_CHAINED|DSF_SIGNATURE|DSF_CORPUS); + ctx->signature = &sig2; + dspam_process(ctx,NULL); + free(sig2.data); + fail_unless(ctx->totals.total_spam == 1,"total spams not 1"); + fail_unless(ctx->totals.total_innocent == 0,"total innocent not 0"); + fail_unless(ctx->totals.spam_misclassified == 0,"total misses not 0"); + fail_unless(ctx->totals.innocent_misclassified == 0,"total fp not 0"); + fail_unless(ctx->totals.spam_corpusfed == 1,"total spam corpus not 1"); + fail_unless(ctx->totals.innocent_corpusfed == 0,"total innoc corpus not 0"); + dspam_destroy(ctx); + /* not really a false positive with CORPUS flag, but... */ + #if API == 28 + ctx = dspam_init(fname, + DSM_PROCESS,DSF_CHAINED|DSF_SIGNATURE|DSF_CORPUS); + #else + ctx = dspam_init(fname, + DSM_FALSEPOSITIVE,DSF_CHAINED|DSF_SIGNATURE|DSF_CORPUS); + #endif + ctx->signature = &sig1; + dspam_process(ctx,NULL); + free(sig1.data); + fail_unless(ctx->totals.total_spam == 1,0); + fail_unless(ctx->totals.total_innocent == 1,0); + fail_unless(ctx->totals.spam_misclassified == 0,0); + fail_unless(ctx->totals.innocent_misclassified == 0,0); + fail_unless(ctx->totals.spam_corpusfed == 1,0); + fail_unless(ctx->totals.innocent_corpusfed == 1,0); + dspam_destroy(ctx); + } END_TEST + + START_TEST(test_reverse) { + DSPAM_CTX *ctx; + struct _ds_spam_signature sig1,sig2; /* signature objects */ + resetuser(fname); + ctx = dspam_init(fname,DSM_PROCESS,DSF_CHAINED|DSF_SIGNATURE); + dspam_process(ctx,msg1); + sig1 = *ctx->signature; ctx->signature->data = NULL; + dspam_destroy(ctx); + ctx = dspam_init(fname,DSM_PROCESS,DSF_CHAINED|DSF_SIGNATURE); + dspam_process(ctx,spam1); + sig2 = *ctx->signature; ctx->signature->data = NULL; + fail_unless(ctx->totals.total_spam == 0,0); + fail_unless(ctx->totals.total_innocent == 2,0); + fail_unless(ctx->totals.spam_misses == 0,0); + fail_unless(ctx->totals.false_positives == 0,0); + dspam_destroy(ctx); + /* change our mind about spam1 */ + ctx = dspam_init(fname,DSM_ADDSPAM,DSF_CHAINED|DSF_SIGNATURE); + ctx->signature = &sig2; + dspam_process(ctx,0); + fail_unless(ctx->totals.total_spam == 1,0); + fail_unless(ctx->totals.total_innocent == 1,0); + fail_unless(ctx->totals.spam_misses == 1,0); + fail_unless(ctx->totals.false_positives == 0,0); + dspam_destroy(ctx); + /* change our mind again */ + ctx = dspam_init(fname,DSM_FALSEPOSITIVE,DSF_CHAINED); + dspam_process(ctx,spam1); + fail_unless(ctx->totals.total_spam == 0,0); + fail_unless(ctx->totals.total_innocent == 2,0); + fail_unless(ctx->totals.spam_misses == 1,0); + fail_unless(ctx->totals.false_positives == 1,0); + dspam_destroy(ctx); + /* and change our mind about msg1 */ + ctx = dspam_init(fname,DSM_ADDSPAM,DSF_CHAINED); + dspam_process(ctx,msg1); + fail_unless(ctx->totals.total_spam == 1,0); + fail_unless(ctx->totals.total_innocent == 1,0); + fail_unless(ctx->totals.spam_misses == 2,0); + fail_unless(ctx->totals.false_positives == 1,0); + dspam_destroy(ctx); + /* test adding a signature as a corpus */ + ctx = dspam_init(fname,DSM_ADDSPAM,DSF_CHAINED|DSF_SIGNATURE|DSF_CORPUS); + ctx->signature = &sig1; + dspam_process(ctx,0); + fail_unless(ctx->totals.total_spam == 2,0); + fail_unless(ctx->totals.total_innocent == 1,0); + fail_unless(ctx->totals.spam_misses == 2,0); + fail_unless(ctx->totals.false_positives == 1,0); + fail_unless(ctx->totals.spam_corpusfed == 1,0); + fail_unless(ctx->totals.innocent_corpusfed == 0,0); + dspam_destroy(ctx); + + free(sig1.data); + free(sig2.data); + } END_TEST + + /* Check that quoted printable encoded attachments are tokenized + * the same as unencoded. */ + static const char msg_7bit[] = "\ + From user@domain.com\n\ + Subject: Test message\n\ + To: testsys\n\ + Content-Type: text/plain; charset=\"us-ascii\"\n\ + Content-Transfer-Encoding: 7bit\n\ + \n\ + Testing 1 2 3\n\ + "; + + static const char msg_quopri[] = "\ + From user@domain.com\n\ + Subject: Test message\n\ + To: testsys\n\ + Content-Type: text/plain; charset=\"us-ascii\"\n\ + Content-Transfer-Encoding: quoted-printable\n\ + \n\ + T=65st=\n\ + ing 1 2 3\n\ + "; + + static const char msg_base64[] = "\ + From user@domain.com\n\ + Subject: Test message\n\ + To: testsys\n\ + Content-Type: text/plain; charset=\"us-ascii\"\n\ + Content-Transfer-Encoding: base64\n\ + \n\ + VGVzdGluZyAxIDIgMwo= + "; + + START_TEST(test_encoding) { + DSPAM_CTX *ctx; + struct _ds_spam_signature sig1,sig2,sig3; /* signature objects */ + resetuser(fname); + ctx = dspam_init(fname,DSM_CLASSIFY, + DSF_CHAINED|DSF_SIGNATURE|DSF_IGNOREHEADER); + dspam_process(ctx,msg_7bit); + sig1 = *ctx->signature; ctx->signature->data = NULL; + dspam_destroy(ctx); + ctx = dspam_init(fname,DSM_CLASSIFY, + DSF_CHAINED|DSF_SIGNATURE|DSF_IGNOREHEADER); + dspam_process(ctx,msg_quopri); + sig2 = *ctx->signature; ctx->signature->data = NULL; + dspam_destroy(ctx); + ctx = dspam_init(fname,DSM_CLASSIFY, + DSF_CHAINED|DSF_SIGNATURE|DSF_IGNOREHEADER); + dspam_process(ctx,msg_base64); + sig3 = *ctx->signature; ctx->signature->data = NULL; + fail_unless(compare_sig(&sig3,&sig1) == 0, "base64 decode failed"); + fail_unless(compare_sig(&sig2,&sig1) == 0, "quopri decode failed"); + free(sig1.data); + free(sig2.data); + free(sig3.data); + dspam_destroy(ctx); + } END_TEST + + /* Check that we do not try to tokenize media attachments. */ + + static const char msg_media1[] = "\ + Subject: Shipments 1099 and 1103 Benderson \n\ + To: Pina.Coloda@dada.com\n\ + X-Mailer: Lotus Notes Release 5.0.9a January 7, 2002\n\ + From: Borealis.Hernandez@dada.com\n\ + Date: Sat, 8 Nov 2003 12:33:44 -0300\n\ + 2003) at 11/08/2003 10:51:13 AM\n\ + MIME-Version: 1.0\n\ + Content-type: multipart/mixed; \n\ + Boundary=\"0__=8CBBE74BDFC6CB808f9e8a93df938690918c8CBBE74BDFC6CB80\"\n\ + Content-Disposition: inline\n\ + \n\ + --0__=8CBBE74BDFC6CB808f9e8a93df938690918c8CBBE74BDFC6CB80\n\ + Content-type: text/plain; charset=us-ascii\n\ + \n\ + I'm sending the following invoices\n\ + \n\ + \n\ + --0__=8CBBE74BDFC6CB808f9e8a93df938690918c8CBBE74BDFC6CB80\n\ + Content-type: application/pdf; \n\ + name=\"Shipments 1099 to 1103 Benderson.pdf\"\n\ + Content-Disposition: attachment;\n\ + filename=\"Shipments 1099 to 1103 Benderson.pdf\"\n\ + Content-transfer-encoding: base64\n\ + \n\ + JVBERi0xLjQNJeLjz9MNCjEgMCBvYmoNPDwgDS9UeXBlIC9DYXRhbG9nIA0vUGFnZXMgMiAwIFIg\n\ + OTk1YTY5MWExPl0NPj4Nc3RhcnR4cmVmDTI0MDk5ODQNJSVFT0YN\n\ + \n\ + --0__=8CBBE74BDFC6CB808f9e8a93df938690918c8CBBE74BDFC6CB80--\n\ + \n\ + "; + + static const char msg_media2[] = "\ + Subject: Shipments 1099 and 1103 Benderson \n\ + To: Pina.Coloda@dada.com\n\ + X-Mailer: Lotus Notes Release 5.0.9a January 7, 2002\n\ + From: Borealis.Hernandez@dada.com\n\ + Date: Sat, 8 Nov 2003 12:33:44 -0300\n\ + 2003) at 11/08/2003 10:51:13 AM\n\ + MIME-Version: 1.0\n\ + Content-type: multipart/mixed; \n\ + Boundary=\"0__=8CBBE74BDFC6CB808f9e8a93df938690918c8CBBE74BDFC6CB80\"\n\ + Content-Disposition: inline\n\ + \n\ + --0__=8CBBE74BDFC6CB808f9e8a93df938690918c8CBBE74BDFC6CB80\n\ + Content-type: text/plain; charset=us-ascii\n\ + \n\ + I'm sending the following invoices\n\ + \n\ + \n\ + --0__=8CBBE74BDFC6CB808f9e8a93df938690918c8CBBE74BDFC6CB80\n\ + Content-type: application/pdf; \n\ + name=\"Shipments 1099 to 1103 Benderson.pdf\"\n\ + Content-Disposition: attachment;\n\ + filename=\"Shipments 1099 to 1103 Benderson.pdf\"\n\ + Content-transfer-encoding: base64\n\ + \n\ + JVBERi0xLjQNJeLjz9MNCjEgMCBvYmfjagofyasdfXBlIC9DYXRhbG9nIA0vUGFnZXMgMiAwIFIg\n\ + kfhgkFJKGOFLG75484950439FHDLKFLKFkglkglkasdfg789g9fhbG9nIA0vUGFnZXMgMiAwIFIg\n\ + OTk1YTY5MWExPl0NPj4Nc3RhcnR4cmVmDTI0MDk5ODQNJSVFT0YN\n\ + \n\ + --0__=8CBBE74BDFC6CB808f9e8a93df938690918c8CBBE74BDFC6CB80--\n\ + \n\ + "; + + START_TEST(test_mediaskip) { + DSPAM_CTX *ctx; + struct _ds_spam_signature sig1,sig2; /* signature objects */ + resetuser(fname); + ctx = dspam_init(fname,DSM_CLASSIFY, + DSF_CHAINED|DSF_SIGNATURE|DSF_IGNOREHEADER); + dspam_process(ctx,msg_media1); + sig1 = *ctx->signature; ctx->signature->data = NULL; + dspam_destroy(ctx); + ctx = dspam_init(fname,DSM_CLASSIFY, + DSF_CHAINED|DSF_SIGNATURE|DSF_IGNOREHEADER); + fail_unless(ctx != 0,0); + dspam_process(ctx,msg_media2); + sig2 = *ctx->signature; ctx->signature->data = NULL; + dspam_destroy(ctx); + /* The two media msgs differ only in the media attachment, so + * the signatures should be identical. */ + fail_unless(compare_sig(&sig2,&sig1)==0,"media skip failed"); + free(sig1.data); + free(sig2.data); + } END_TEST + + /* Check that HTML comments do not split tokens. */ + + static const char msg_html1[] = "\ + From user@domain.com\n\ + Subject: Test message\n\ + To: testsys\n\ + Content-Type: text/html; charset=\"us-ascii\"\n\ + Content-Transfer-Encoding: 7bit\n\ + \n\ + \n\ + Buy our prescription Viagra!\n\ + \n\ + "; + + static const char msg_html2[] = "\ + From user@domain.com\n\ + Subject: Test message\n\ + To: testsys\n\ + Content-Type: text/html; charset=\"us-ascii\"\n\ + Content-Transfer-Encoding: 7bit\n\ + \n\ + \n\ + Buy our prescription Viagra!\n\ + \n\ + "; + + START_TEST(test_html) { + DSPAM_CTX *ctx; + struct _ds_spam_signature sig1,sig2; /* signature objects */ + resetuser(fname); + ctx = dspam_init(fname,DSM_CLASSIFY,DSF_CHAINED|DSF_SIGNATURE); + dspam_process(ctx,msg_html1); + sig1 = *ctx->signature; ctx->signature->data = NULL; + dspam_destroy(ctx); + ctx = dspam_init(fname,DSM_CLASSIFY,DSF_CHAINED|DSF_SIGNATURE); + dspam_process(ctx,msg_html2); + sig2 = *ctx->signature; ctx->signature->data = NULL; + fail_unless(sig1.length == sig2.length + && memcmp(sig2.data,sig1.data,sig1.length) == 0, + "HTML comment stripping failed"); + free(sig1.data); + free(sig2.data); + dspam_destroy(ctx); + } END_TEST + + static double eps = 0.0000001; + + static void verify_tbt(struct tbt *tbt,int items) { + double delta = 1.0; + int cnt = 0; + struct tbt_node *node = tbt_first(tbt); + fail_unless(tbt->items == items,"tbt_add lost items"); + while (node) { + //fprintf(stderr,"delta = %g\n",delta); + fail_unless(node->delta < delta + eps,"deltas not in descending order"); + delta = node->delta; + ++cnt; + node = tbt_next(node); + } + fail_unless(cnt == items,"tbt sort lost items"); + } + + /* test token delta sorting */ + START_TEST(test_tbt) { + struct tbt *tbt = tbt_create(); + unsigned long long crc = 0; + char buf[80]; + int i; + srandom(5551212L); + for (i = 0; i < 5000; ++i) { + double prob = (double)random() / (double)RAND_MAX; + fail_unless(prob <= 1.0 && prob >= 0.0,"problem with random() or RAND_MAX"); + tbt_add(tbt,prob,++crc,1); + } + verify_tbt(tbt,5000); + i = tbt_destroy(tbt); + sprintf(buf,"tbt_destroy returned %d",i); + fail_unless(i == 0,buf); + + tbt = tbt_create(); + /* worst case is that all tokens have equal delta. */ + for (i = 0; i < 2000; ++i) tbt_add(tbt,0.7,++crc,1); + for (i = 0; i < 2000; ++i) tbt_add(tbt,0.3,++crc,1); + verify_tbt(tbt,4000); + i = tbt_destroy(tbt); + sprintf(buf,"tbt_destroy returned %d",i); + fail_unless(i == 0,buf); + } END_TEST + + #ifdef TEST_TOKENIZE + + static struct lht * + tokenize(int chained,const char *msg) { + char *edup = strdup(msg); + char *p; + struct lht *freq; + if (edup == 0) return 0; + p = strstr(edup,"\n\n"); + if (p) { + *p++ = 0; + freq = _ds_tokenize(chained,edup,p); + } + else + freq = _ds_tokenize(chained," ",edup); + free(edup); + return freq; + } + + /* tokenize a simple message */ + START_TEST(test_tokenize) { + struct lht *freq; + struct lht_node *node_lht; + struct lht_c c_lht; + int tokens = 0; + + freq = tokenize(1,nasty1); + fail_unless(freq != 0,"out of memory"); + node_lht = c_lht_first(freq, &c_lht); + while (node_lht != NULL) { + char buf[256]; + sprintf(buf,"%s: %d\n",node_lht->token_name,node_lht->frequency); + if (strcmp("Ha",node_lht->token_name) == 0) + fail_unless(node_lht->frequency == 3,buf); + else if (strcmp("Ha+Ha",node_lht->token_name) == 0) + fail_unless(node_lht->frequency == 2,buf); + else + fail_unless(node_lht->frequency == 1,buf); + tokens += node_lht->frequency; + node_lht = c_lht_next(freq, &c_lht); + } + fail_unless(tokens == 32,"token count not 32"); + lht_destroy(freq); + fflush(stdout); + } END_TEST + #endif + + /* Collect all the tests. This will make more sense when tests are + * in multiple source files. */ + Suite *dspam_suite (void) { + Suite *s = suite_create ("DSPAM"); + TCase *tc_process = tcase_create ("PROCESS"); + + suite_add_tcase (s, tc_process); + tcase_add_test (tc_process, test_corpus); + tcase_add_test (tc_process, test_classify); + tcase_add_test (tc_process, test_overflow); + #ifdef TEST_TOKENIZE + tcase_add_test (tc_process, test_tokenize); + #endif + tcase_add_test (tc_process, test_reverse); + tcase_add_test (tc_process, test_encoding); + tcase_add_test (tc_process, test_mediaskip); + tcase_add_test (tc_process, test_html); + tcase_add_test (tc_process, test_tbt); + #if 0 && DSPAM_API == 28 + tcase_add_checked_fixture (tc_process, + dspam_init_driver,dspam_shutdown_driver); + #endif + return s; + } + + int main (void) { + int nf; + Suite *s = dspam_suite (); + SRunner *sr = srunner_create (s); + dspam_init_driver(); + srunner_run_all (sr, CK_NORMAL); + dspam_shutdown_driver(); + nf = srunner_ntests_failed (sr); + srunner_free (sr); + suite_free (s); + return (nf == 0) ? EXIT_SUCCESS : EXIT_FAILURE; + } Index: dspam/util.c diff -c dspam/util.c:1.1.1.5 dspam/util.c:1.1.1.5.2.1 *** dspam/util.c:1.1.1.5 Tue Oct 28 16:06:14 2003 --- dspam/util.c Sat Nov 15 18:34:58 2003 *************** *** 205,217 **** /* Locks use USERDIR */ if (extension != NULL && !strcmp (extension, "lock")) { ! snprintf (path, sizeof (path), "%s/%s/%s.%s", USERDIR, filename, filename, extension); return path; } #endif #ifdef HOMEDIR ! if (extension != NULL && (!strcmp (extension, "nodspam") || !strcmp (extension, "dspam") || !strcmp (extension, "inoc"))) { struct passwd *p; p = getpwnam (filename); --- 205,220 ---- /* Locks use USERDIR */ if (extension != NULL && !strcmp (extension, "lock")) { ! snprintf (path, sizeof (path), "%s/%s/%s.%s", USERDIR, filename, filename, ! extension); return path; } #endif #ifdef HOMEDIR ! if (extension != NULL ! && (!strcmp (extension, "nodspam") || !strcmp (extension, "dspam") ! || !strcmp (extension, "inoc"))) { struct passwd *p; p = getpwnam (filename); *************** *** 229,252 **** #ifdef LARGESCALE if (filename[1] != 0) { ! if (extension == NULL) { ! snprintf (path, MAX_FILENAME_LENGTH, "%s/%c/%c/%s", ! USERDIR, filename[0], filename[1], filename); ! } else { if (extension[0] == 0) snprintf (path, MAX_FILENAME_LENGTH, "%s/%c/%c/%s/%s", USERDIR, filename[0], filename[1], filename, filename); else snprintf (path, MAX_FILENAME_LENGTH, "%s/%c/%c/%s/%s.%s", ! USERDIR, filename[0], filename[1], filename, filename, extension); } } else { ! if (extension == NULL) { ! snprintf (path, MAX_FILENAME_LENGTH, "%s/%c/%s", ! USERDIR, filename[0], filename); ! } else { if (extension[0] == 0) snprintf (path, MAX_FILENAME_LENGTH, "%s/%c/%s/%s", USERDIR, filename[0], filename, filename); --- 232,262 ---- #ifdef LARGESCALE if (filename[1] != 0) { ! if (extension == NULL) ! { ! snprintf (path, MAX_FILENAME_LENGTH, "%s/%c/%c/%s", ! USERDIR, filename[0], filename[1], filename); ! } ! else ! { if (extension[0] == 0) snprintf (path, MAX_FILENAME_LENGTH, "%s/%c/%c/%s/%s", USERDIR, filename[0], filename[1], filename, filename); else snprintf (path, MAX_FILENAME_LENGTH, "%s/%c/%c/%s/%s.%s", ! USERDIR, filename[0], filename[1], filename, filename, ! extension); } } else { ! if (extension == NULL) ! { ! snprintf (path, MAX_FILENAME_LENGTH, "%s/%c/%s", ! USERDIR, filename[0], filename); ! } ! else ! { if (extension[0] == 0) snprintf (path, MAX_FILENAME_LENGTH, "%s/%c/%s/%s", USERDIR, filename[0], filename, filename); *************** *** 256,266 **** } } #else ! if (extension == NULL) { snprintf (path, MAX_FILENAME_LENGTH, "%s/%s", USERDIR, filename); ! } else { snprintf (path, MAX_FILENAME_LENGTH, "%s/%s/%s.%s", ! USERDIR, filename, filename, extension); } #endif --- 266,279 ---- } } #else ! if (extension == NULL) ! { snprintf (path, MAX_FILENAME_LENGTH, "%s/%s", USERDIR, filename); ! } ! else ! { snprintf (path, MAX_FILENAME_LENGTH, "%s/%s/%s.%s", ! USERDIR, filename, filename, extension); } #endif *************** *** 278,284 **** if (!filename) { LOG (LOG_ERR, ! "_ds_prepare_path_for(): invalid argument: filename == NULL"); return EUNKNOWN; } --- 291,297 ---- if (!filename) { LOG (LOG_ERR, ! "_ds_prepare_path_for(): invalid argument: filename == NULL"); return EUNKNOWN; } *************** *** 307,313 **** if (x) { file_error (ERROR_DIR_CREATE, path, strerror (errno)); ! free (file_buffer_start); return EFILE; } } --- 320,326 ---- if (x) { file_error (ERROR_DIR_CREATE, path, strerror (errno)); ! free (file_buffer_start); return EFILE; } } Index: dspam/tools/dspam_2mysql.c diff -c dspam/tools/dspam_2mysql.c:1.1.1.1 dspam/tools/dspam_2mysql.c:1.1.1.1.2.1 *** dspam/tools/dspam_2mysql.c:1.1.1.1 Wed Oct 29 05:34:07 2003 --- dspam/tools/dspam_2mysql.c Sat Nov 15 18:34:58 2003 *************** *** 54,66 **** open_ctx = open_mtx = NULL; ! signal(SIGINT, dieout); ! signal(SIGPIPE, dieout); ! signal(SIGTERM, dieout); ! dspam_init_driver(); exit (process_all_users ()); ! dspam_shutdown_driver(); } int --- 54,66 ---- open_ctx = open_mtx = NULL; ! signal (SIGINT, dieout); ! signal (SIGPIPE, dieout); ! signal (SIGTERM, dieout); ! dspam_init_driver (); exit (process_all_users ()); ! dspam_shutdown_driver (); } int *************** *** 114,120 **** printf ("insert into dspam_stats (uid, total_spam, total_innocent, spam_misses, false_positives, spam_corpusfed, innocent_corpusfed) values(%d, %ld, %ld, %ld, %ld, %ld, %ld);\n", (int) p->pw_uid, CTX->totals.total_spam, CTX->totals.total_innocent, ! CTX->totals.spam_misclassified, CTX->totals.innocent_misclassified, CTX->totals.spam_corpusfed, CTX->totals.innocent_corpusfed); record = _ds_get_nexttoken (CTX); --- 114,120 ---- printf ("insert into dspam_stats (uid, total_spam, total_innocent, spam_misses, false_positives, spam_corpusfed, innocent_corpusfed) values(%d, %ld, %ld, %ld, %ld, %ld, %ld);\n", (int) p->pw_uid, CTX->totals.total_spam, CTX->totals.total_innocent, ! CTX->totals.spam_misclassified, CTX->totals.innocent_misclassified, CTX->totals.spam_corpusfed, CTX->totals.innocent_corpusfed); record = _ds_get_nexttoken (CTX); *************** *** 122,129 **** { printf ("insert into dspam_token_data (uid, token, spam_hits, innocent_hits, last_hit) values(%d, %llu, %ld, %ld, %ld);\n", ! (int) p->pw_uid, record->token, record->spam_hits, record->innocent_hits, ! record->last_hit); record = _ds_get_nexttoken (CTX); } --- 122,129 ---- { printf ("insert into dspam_token_data (uid, token, spam_hits, innocent_hits, last_hit) values(%d, %llu, %ld, %ld, %ld);\n", ! (int) p->pw_uid, record->token, record->spam_hits, ! record->innocent_hits, record->last_hit); record = _ds_get_nexttoken (CTX); } *************** *** 132,143 **** return 0; } ! void dieout(int signal) { ! fprintf(stderr, "terminated.\n"); if (open_ctx != NULL) ! dspam_destroy(open_ctx); if (open_mtx != NULL) ! dspam_destroy(open_mtx); ! exit(EXIT_SUCCESS); } - --- 132,144 ---- return 0; } ! void ! dieout (int signal) ! { ! fprintf (stderr, "terminated.\n"); if (open_ctx != NULL) ! dspam_destroy (open_ctx); if (open_mtx != NULL) ! dspam_destroy (open_mtx); ! exit (EXIT_SUCCESS); } Index: dspam/tools/dspam_clean.c diff -c dspam/tools/dspam_clean.c:1.1.1.7 dspam/tools/dspam_clean.c:1.1.1.7.2.1 *** dspam/tools/dspam_clean.c:1.1.1.7 Sun Oct 26 17:21:52 2003 --- dspam/tools/dspam_clean.c Sat Nov 15 18:34:58 2003 *************** *** 69,86 **** open_ctx = open_mtx = NULL; ! signal(SIGINT, dieout); ! signal(SIGPIPE, dieout); ! signal(SIGTERM, dieout); ! ! dspam_init_driver(); CTX = dspam_init (NULL, NULL, DSM_TOOLS, 0); open_ctx = CTX; if (CTX == NULL) { fprintf (stderr, "Could not initialize context: %s\n", strerror (errno)); ! dspam_shutdown_driver(); exit (EXIT_FAILURE); } --- 69,86 ---- open_ctx = open_mtx = NULL; ! signal (SIGINT, dieout); ! signal (SIGPIPE, dieout); ! signal (SIGTERM, dieout); ! ! dspam_init_driver (); CTX = dspam_init (NULL, NULL, DSM_TOOLS, 0); open_ctx = CTX; if (CTX == NULL) { fprintf (stderr, "Could not initialize context: %s\n", strerror (errno)); ! dspam_shutdown_driver (); exit (EXIT_FAILURE); } *************** *** 108,114 **** dspam_destroy (CTX); open_ctx = NULL; ! dspam_shutdown_driver(); exit (EXIT_SUCCESS); } --- 108,114 ---- dspam_destroy (CTX); open_ctx = NULL; ! dspam_shutdown_driver (); exit (EXIT_SUCCESS); } *************** *** 139,150 **** return 0; } ! void dieout(int signal) { ! fprintf(stderr, "terminated.\n"); if (open_ctx != NULL) ! dspam_destroy(open_ctx); if (open_mtx != NULL) ! dspam_destroy(open_mtx); ! exit(EXIT_SUCCESS); } - --- 139,151 ---- return 0; } ! void ! dieout (int signal) ! { ! fprintf (stderr, "terminated.\n"); if (open_ctx != NULL) ! dspam_destroy (open_ctx); if (open_mtx != NULL) ! dspam_destroy (open_mtx); ! exit (EXIT_SUCCESS); } Index: dspam/tools/dspam_dump.c diff -c dspam/tools/dspam_dump.c:1.1.1.6 dspam/tools/dspam_dump.c:1.1.1.6.2.1 *** dspam/tools/dspam_dump.c:1.1.1.6 Wed Oct 29 05:34:34 2003 --- dspam/tools/dspam_dump.c Sat Nov 15 18:34:58 2003 *************** *** 65,73 **** int r; open_ctx = NULL; ! signal(SIGINT, dieout); ! signal(SIGPIPE, dieout); ! signal(SIGTERM, dieout); if (argc < 2) { --- 65,73 ---- int r; open_ctx = NULL; ! signal (SIGINT, dieout); ! signal (SIGPIPE, dieout); ! signal (SIGTERM, dieout); if (argc < 2) { *************** *** 75,94 **** exit (EXIT_FAILURE); } ! dspam_init_driver(); CTX = dspam_init (argv[1], NULL, DSM_CLASSIFY, 0); open_ctx = CTX; if (CTX == NULL) { fprintf (stderr, "Could not init context: %s\n", strerror (errno)); ! dspam_shutdown_driver(); exit (EXIT_FAILURE); } r = dump_database (CTX); ! dspam_destroy(CTX); open_ctx = NULL; ! dspam_shutdown_driver(); return (r) ? EXIT_FAILURE : EXIT_SUCCESS; } --- 75,94 ---- exit (EXIT_FAILURE); } ! dspam_init_driver (); CTX = dspam_init (argv[1], NULL, DSM_CLASSIFY, 0); open_ctx = CTX; if (CTX == NULL) { fprintf (stderr, "Could not init context: %s\n", strerror (errno)); ! dspam_shutdown_driver (); exit (EXIT_FAILURE); } r = dump_database (CTX); ! dspam_destroy (CTX); open_ctx = NULL; ! dspam_shutdown_driver (); return (r) ? EXIT_FAILURE : EXIT_SUCCESS; } *************** *** 109,118 **** return 0; } ! void dieout(int signal) { ! fprintf(stderr, "terminated.\n"); if (open_ctx != NULL) ! dspam_destroy(open_ctx); exit (EXIT_SUCCESS); } - --- 109,119 ---- return 0; } ! void ! dieout (int signal) ! { ! fprintf (stderr, "terminated.\n"); if (open_ctx != NULL) ! dspam_destroy (open_ctx); exit (EXIT_SUCCESS); } Index: dspam/tools/dspam_merge.c diff -c dspam/tools/dspam_merge.c:1.1.1.1 dspam/tools/dspam_merge.c:1.1.1.1.2.1 *** dspam/tools/dspam_merge.c:1.1.1.1 Thu Oct 30 00:04:15 2003 --- dspam/tools/dspam_merge.c Sat Nov 15 18:34:58 2003 *************** *** 66,76 **** } open_ctx = open_mtx = NULL; ! signal(SIGINT, dieout); ! signal(SIGPIPE, dieout); ! signal(SIGTERM, dieout); ! dspam_init_driver(); users = nt_create (NT_CHAR); freq = lht_create (1543); --- 66,76 ---- } open_ctx = open_mtx = NULL; ! signal (SIGINT, dieout); ! signal (SIGPIPE, dieout); ! signal (SIGTERM, dieout); ! dspam_init_driver (); users = nt_create (NT_CHAR); freq = lht_create (1543); *************** *** 99,105 **** bail: nt_destroy (users); lht_destroy (freq); ! dspam_shutdown_driver(); exit (EXIT_FAILURE); } --- 99,105 ---- bail: nt_destroy (users); lht_destroy (freq); ! dspam_shutdown_driver (); exit (EXIT_FAILURE); } *************** *** 109,116 **** #ifdef DEBUG printf ("Merging user: %s\n", (const char *) node_nt->ptr); #endif ! MTX = dspam_init ((const char *) node_nt->ptr, NULL, ! DSM_CLASSIFY, 0); open_mtx = MTX; if (MTX == NULL) { --- 109,115 ---- #ifdef DEBUG printf ("Merging user: %s\n", (const char *) node_nt->ptr); #endif ! MTX = dspam_init ((const char *) node_nt->ptr, NULL, DSM_CLASSIFY, 0); open_mtx = MTX; if (MTX == NULL) { *************** *** 149,178 **** printf ("processed %ld tokens\n", i); #endif node_nt = c_nt_next (users, &c_nt); ! dspam_destroy(MTX); open_mtx = NULL; } #ifdef DEBUG ! printf("storing merged tokens...\n"); #endif _ds_setall_spamrecords (CTX, freq); #ifdef DEBUG ! printf("completed.\n"); #endif nt_destroy (users); lht_destroy (freq); dspam_destroy (CTX); open_ctx = NULL; ! dspam_shutdown_driver(); exit (EXIT_SUCCESS); } ! void dieout(int signal) { ! fprintf(stderr, "terminated.\n"); if (open_ctx != NULL) ! dspam_destroy(open_ctx); if (open_mtx != NULL) ! dspam_destroy(open_mtx); ! exit(EXIT_SUCCESS); } --- 148,179 ---- printf ("processed %ld tokens\n", i); #endif node_nt = c_nt_next (users, &c_nt); ! dspam_destroy (MTX); open_mtx = NULL; } #ifdef DEBUG ! printf ("storing merged tokens...\n"); #endif _ds_setall_spamrecords (CTX, freq); #ifdef DEBUG ! printf ("completed.\n"); #endif nt_destroy (users); lht_destroy (freq); dspam_destroy (CTX); open_ctx = NULL; ! dspam_shutdown_driver (); exit (EXIT_SUCCESS); } ! void ! dieout (int signal) ! { ! fprintf (stderr, "terminated.\n"); if (open_ctx != NULL) ! dspam_destroy (open_ctx); if (open_mtx != NULL) ! dspam_destroy (open_mtx); ! exit (EXIT_SUCCESS); } Index: dspam/tools/dspam_ngstats.c diff -c dspam/tools/dspam_ngstats.c:1.1.1.1 dspam/tools/dspam_ngstats.c:1.1.1.1.2.1 *** dspam/tools/dspam_ngstats.c:1.1.1.1 Sat Nov 8 13:39:32 2003 --- dspam/tools/dspam_ngstats.c Sat Nov 15 18:34:58 2003 *************** *** 64,70 **** int sock; char get[1024]; ! dspam_init_driver(); process_all_users (); --- 64,70 ---- int sock; char get[1024]; ! dspam_init_driver (); process_all_users (); *************** *** 80,86 **** { fprintf (stderr, "unable to connect to %s: %s\n", NGSTATS_HOST, strerror (errno)); ! dspam_shutdown_driver(); exit (EXIT_FAILURE); } --- 80,86 ---- { fprintf (stderr, "unable to connect to %s: %s\n", NGSTATS_HOST, strerror (errno)); ! dspam_shutdown_driver (); exit (EXIT_FAILURE); } *************** *** 99,105 **** close (sock); } ! dspam_shutdown_driver(); exit (EXIT_SUCCESS); } --- 99,105 ---- close (sock); } ! dspam_shutdown_driver (); exit (EXIT_SUCCESS); } Index: dspam/tools/dspam_stats.c diff -c dspam/tools/dspam_stats.c:1.1.1.7 dspam/tools/dspam_stats.c:1.1.1.7.2.1 *** dspam/tools/dspam_stats.c:1.1.1.7 Thu Oct 30 11:20:34 2003 --- dspam/tools/dspam_stats.c Sat Nov 15 18:34:58 2003 *************** *** 55,65 **** open_ctx = open_mtx = NULL; ! signal(SIGINT, dieout); ! signal(SIGPIPE, dieout); ! signal(SIGTERM, dieout); ! ! dspam_init_driver(); if (argc > 1) { --- 55,65 ---- open_ctx = open_mtx = NULL; ! signal (SIGINT, dieout); ! signal (SIGPIPE, dieout); ! signal (SIGTERM, dieout); ! ! dspam_init_driver (); if (argc > 1) { *************** *** 70,76 **** process_all_users (); } ! dspam_shutdown_driver(); exit (EXIT_SUCCESS); } --- 70,76 ---- process_all_users (); } ! dspam_shutdown_driver (); exit (EXIT_SUCCESS); } *************** *** 113,135 **** return EUNKNOWN; } ! printf ("%-16s TS:% 6ld TI:% 6ld SM:% 6ld IM:% 6ld SC:% 6ld IC:% 6ld\n", username, ! MTX->totals.total_spam - (MTX->totals.spam_misclassified + MTX->totals.spam_corpusfed), ! MTX->totals.total_innocent - (MTX->totals.innocent_misclassified + MTX->totals.innocent_corpusfed), ! MTX->totals.spam_misclassified, MTX->totals.innocent_misclassified, ! MTX->totals.spam_corpusfed, MTX->totals.innocent_corpusfed); dspam_destroy (MTX); open_mtx = NULL; return 0; } ! void dieout(int signal) { ! fprintf(stderr, "terminated.\n"); if (open_ctx != NULL) ! dspam_destroy(open_ctx); if (open_mtx != NULL) ! dspam_destroy(open_mtx); ! exit(EXIT_SUCCESS); } - --- 113,139 ---- return EUNKNOWN; } ! printf ("%-16s TS:% 6ld TI:% 6ld SM:% 6ld IM:% 6ld SC:% 6ld IC:% 6ld\n", ! username, ! MTX->totals.total_spam - (MTX->totals.spam_misclassified + ! MTX->totals.spam_corpusfed), ! MTX->totals.total_innocent - (MTX->totals.innocent_misclassified + ! MTX->totals.innocent_corpusfed), ! MTX->totals.spam_misclassified, MTX->totals.innocent_misclassified, ! MTX->totals.spam_corpusfed, MTX->totals.innocent_corpusfed); dspam_destroy (MTX); open_mtx = NULL; return 0; } ! void ! dieout (int signal) ! { ! fprintf (stderr, "terminated.\n"); if (open_ctx != NULL) ! dspam_destroy (open_ctx); if (open_mtx != NULL) ! dspam_destroy (open_mtx); ! exit (EXIT_SUCCESS); } Index: dspam/tools.libdb3_drv/libdb3_purge.c diff -c dspam/tools.libdb3_drv/libdb3_purge.c:1.1.1.1 dspam/tools.libdb3_drv/libdb3_purge.c:1.1.1.1.2.1 *** dspam/tools.libdb3_drv/libdb3_purge.c:1.1.1.1 Wed Oct 29 05:44:49 2003 --- dspam/tools.libdb3_drv/libdb3_purge.c Sat Nov 15 18:34:58 2003 *************** *** 70,78 **** printf ("purge start.\n"); #endif ! dspam_init_driver(); rc = process_directory (USERDIR); ! dspam_shutdown_driver(); return rc < 0 ? EXIT_FAILURE : EXIT_SUCCESS; } --- 70,78 ---- printf ("purge start.\n"); #endif ! dspam_init_driver (); rc = process_directory (USERDIR); ! dspam_shutdown_driver (); return rc < 0 ? EXIT_FAILURE : EXIT_SUCCESS; } *************** *** 139,145 **** x = strstr (user, ".dict"); x[0] = 0; ! strcpy(db_home, _ds_userdir_path(user, NULL)); CTX = dspam_init (user, NULL, DSM_TOOLS, 0); st = (struct _libdb3_drv_storage *) CTX->storage; --- 139,145 ---- x = strstr (user, ".dict"); x[0] = 0; ! strcpy (db_home, _ds_userdir_path (user, NULL)); CTX = dspam_init (user, NULL, DSM_TOOLS, 0); st = (struct _libdb3_drv_storage *) CTX->storage; *************** *** 155,161 **** { fprintf (stderr, "db_env_create failed: %s", db_strerror (result)); rc = EFAILURE; ! dspam_destroy(CTX); continue; } --- 155,161 ---- { fprintf (stderr, "db_env_create failed: %s", db_strerror (result)); rc = EFAILURE; ! dspam_destroy (CTX); continue; } *************** *** 171,177 **** DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL, 0)) != 0) { fprintf (stderr, "DB_ENV->open failed: %s", db_strerror (result)); ! dspam_destroy(CTX); rc = EFAILURE; continue; } --- 171,177 ---- DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL, 0)) != 0) { fprintf (stderr, "DB_ENV->open failed: %s", db_strerror (result)); ! dspam_destroy (CTX); rc = EFAILURE; continue; } *************** *** 181,187 **** { file_error (ERROR_CURSOR_CREATE, filename, db_strerror (ret)); rc = EFAILURE; ! dspam_destroy(CTX); continue; } --- 181,187 ---- { file_error (ERROR_CURSOR_CREATE, filename, db_strerror (ret)); rc = EFAILURE; ! dspam_destroy (CTX); continue; } *************** *** 229,235 **** #endif c->c_close (c); ! dspam_destroy(CTX); } } closedir (dir); --- 229,235 ---- #endif c->c_close (c); ! dspam_destroy (CTX); } } closedir (dir); Index: dspam/tools.libdb4_drv/libdb4_purge.c diff -c dspam/tools.libdb4_drv/libdb4_purge.c:1.1.1.1 dspam/tools.libdb4_drv/libdb4_purge.c:1.1.1.1.2.1 *** dspam/tools.libdb4_drv/libdb4_purge.c:1.1.1.1 Wed Oct 29 05:44:35 2003 --- dspam/tools.libdb4_drv/libdb4_purge.c Sat Nov 15 18:34:58 2003 *************** *** 74,82 **** printf ("purge start.\n"); #endif ! dspam_init_driver(); rc = process_directory (USERDIR); ! dspam_shutdown_driver(); return rc < 0 ? EXIT_FAILURE : EXIT_SUCCESS; } --- 74,82 ---- printf ("purge start.\n"); #endif ! dspam_init_driver (); rc = process_directory (USERDIR); ! dspam_shutdown_driver (); return rc < 0 ? EXIT_FAILURE : EXIT_SUCCESS; } *************** *** 141,147 **** x = strstr (user, ".dict"); x[0] = 0; ! strcpy(db_home, _ds_userdir_path(user, NULL)); CTX = dspam_init (user, NULL, DSM_TOOLS, 0); st = (struct _libdb4_drv_storage *) CTX->storage; --- 141,147 ---- x = strstr (user, ".dict"); x[0] = 0; ! strcpy (db_home, _ds_userdir_path (user, NULL)); CTX = dspam_init (user, NULL, DSM_TOOLS, 0); st = (struct _libdb4_drv_storage *) CTX->storage; *************** *** 156,162 **** { file_error (ERROR_CURSOR_CREATE, file, db_strerror (ret)); rc = EFAILURE; ! dspam_destroy(CTX); continue; } --- 156,162 ---- { file_error (ERROR_CURSOR_CREATE, file, db_strerror (ret)); rc = EFAILURE; ! dspam_destroy (CTX); continue; } *************** *** 191,197 **** if (ret) { unsigned long long token; ! memcpy(&token, key.data, sizeof(unsigned long long)); fprintf (stderr, "delete '%llu' failed %d\n", token, ret); rc = EFAILURE; } --- 191,197 ---- if (ret) { unsigned long long token; ! memcpy (&token, key.data, sizeof (unsigned long long)); fprintf (stderr, "delete '%llu' failed %d\n", token, ret); rc = EFAILURE; } *************** *** 206,216 **** #endif c->c_close (c); #ifdef DEBUG ! fprintf(stderr, "closed\n"); #endif ! dspam_destroy(CTX); #ifdef DEBUG ! fprintf(stderr, "database closed\n"); #endif } } --- 206,216 ---- #endif c->c_close (c); #ifdef DEBUG ! fprintf (stderr, "closed\n"); #endif ! dspam_destroy (CTX); #ifdef DEBUG ! fprintf (stderr, "database closed\n"); #endif } }