DLP updates

git-svn: trunk@3798

ChangeLog

@@ -1,3 +1,12 @@
+Fri Apr 18 18:33:59 CEST 2008 (tk)
+----------------------------------
+  * libclamav: DLP: dconf support; pass flags through scan options
+  * clamd: new options: StructuredDataDetection, StructuredMinCreditCardCount,
+	   StructuredMinSSNCount, StructuredSSNFormatNormal,
+	   StructuredSSNFormatStripped
+  * etc/clamd.conf, docs/man/clamd.conf.5.in: update
+  * TODO: add DLP fine-tuning options to clamscan
+
 Fri Apr 18 13:55:41 EEST 2008 (edwin)
 -------------------------------------
   * libclamav/dconf.h: fix flag code assignment

clamd/server-th.c

@@ -438,6 +438,22 @@ int acceptloop_th(int *socketds, int nsockets, struct cl_engine *engine, unsigne
 	}
     }
 
+    if(cfgopt(copt, "StructuredDataDetection")->enabled) {
+        options |= CL_SCAN_STRUCTURED;
+
+        limits.min_cc_count = cfgopt(copt, "StructuredMinCreditCardCount")->numarg;
+        logg("Structured: Minimum Credit Card Number Count set to %u\n", limits.min_cc_count);
+
+        limits.min_ssn_count = cfgopt(copt, "StructuredMinSSNCount")->numarg;
+        logg("Structured: Minimum Social Security Number Count set to %u\n", limits.min_ssn_count);
+
+        if(cfgopt(copt, "StructuredSSNFormatNormal")->enabled)
+            options |= CL_SCAN_STRUCTURED_SSN_NORMAL;
+
+        if(cfgopt(copt, "StructuredSSNFormatStripped")->enabled)
+	    options |= CL_SCAN_STRUCTURED_SSN_STRIPPED;
+    }
+
     selfchk = cfgopt(copt, "SelfCheck")->numarg;
     if(!selfchk) {
 	logg("Self checking disabled.\n");

clamscan/manager.c

@@ -310,9 +310,10 @@ int scanmanager(const struct optstruct *opt)
 
     if(opt_check(opt, "detect-structured")) {
 	options |= CL_SCAN_STRUCTURED;
+	options |= CL_SCAN_STRUCTURED_SSN_NORMAL;
+	options |= CL_SCAN_STRUCTURED_SSN_STRIPPED;
         limits.min_cc_count = 1;
         limits.min_ssn_count = 1;
-        limits.structured_flags = CL_STRUCTURED_CONF_SSN_BOTH;
     } else
 	options &= ~CL_SCAN_STRUCTURED;
 

docs/man/clamd.conf.5.in

@@ -263,6 +263,31 @@ Always block cloaked URLs, even if URL isn't in database. This can lead to false
 .br
 Default: no
 .TP
+\fBStructuredDataDetection BOOL\fR
+Enable the DLP module.
+.br 
+Default: no
+.TP
+\fBStructuredMinCreditCardCount NUMBER\fR
+This option sets the lowest number of Credit Card numbers found in a file to generate a detect.
+.br 
+Default: 1
+.TP
+\fBStructuredMinSSNCount NUMBER\fR
+This option sets the lowest number of Social Security Numbers found in a file to generate a detect.
+.br 
+Default: 1
+.TP
+\fBStructuredSSNFormatNormal BOOL\fR
+With this option enabled the DLP module will search for valid SSNs formatted as xxx-yy-zzzz.
+.br 
+Default: Yes
+.TP
+\fBStructuredSSNFormatStripped BOOL\fR
+With this option enabled the DLP module will search for valid SSNs formatted as xxxyyzzzz.
+.br 
+Default: Yes
+.TP
 \fBScanArchive BOOL\fR
 Enable archive scanning.
 .br 

etc/clamd.conf

@@ -246,6 +246,35 @@ LocalSocket /tmp/clamd.socket
 #PhishingAlwaysBlockCloak no
 
 
+##
+## Data Loss Prevention (DLP)
+##
+
+# Enable the DLP module
+# Default: No
+#StructuredDataDetection yes
+
+# This option sets the lowest number of Credit Card numbers found in a file
+# to generate a detect.
+# Default: 1
+#StructuredMinCreditCardCount 5
+
+# This option sets the lowest number of Social Security Numbers found
+# in a file to generate a detect.
+# Default: 1
+#StructuredMinSSNCount 5
+
+# With this option enabled the DLP module will search for valid
+# SSNs formatted as xxx-yy-zzzz
+# Default: yes
+#StructuredSSNFormatNormal yes
+
+# With this option enabled the DLP module will search for valid
+# SSNs formatted as xxxyyzzzz
+# Default: yes
+#StructuredSSNFormatStripped yes
+
+
 ##
 ## HTML
 ##

libclamav/clamav.h

@@ -77,23 +77,24 @@ extern "C"
 #define CL_DB_STDOPT	    (CL_DB_PHISHING | CL_DB_PHISHING_URLS)
 
 /* scan options */
-#define CL_SCAN_RAW		    0x0
+#define CL_SCAN_RAW			0x0
-#define CL_SCAN_ARCHIVE		    0x1
+#define CL_SCAN_ARCHIVE			0x1
-#define CL_SCAN_MAIL		    0x2
+#define CL_SCAN_MAIL			0x2
-#define CL_SCAN_OLE2		    0x4
+#define CL_SCAN_OLE2			0x4
-#define CL_SCAN_BLOCKENCRYPTED	    0x8
+#define CL_SCAN_BLOCKENCRYPTED		0x8
-#define CL_SCAN_HTML		    0x10
+#define CL_SCAN_HTML			0x10
-#define CL_SCAN_PE		    0x20
+#define CL_SCAN_PE			0x20
-#define CL_SCAN_BLOCKBROKEN	    0x40
+#define CL_SCAN_BLOCKBROKEN		0x40
-#define CL_SCAN_MAILURL		    0x80
+#define CL_SCAN_MAILURL			0x80
-#define CL_SCAN_BLOCKMAX	    0x100 /* ignored */
+#define CL_SCAN_BLOCKMAX		0x100 /* ignored */
-#define CL_SCAN_ALGORITHMIC	    0x200
+#define CL_SCAN_ALGORITHMIC		0x200
-#define CL_SCAN_PHISHING_BLOCKSSL   0x800 /* ssl mismatches, not ssl by itself*/
+#define CL_SCAN_PHISHING_BLOCKSSL	0x800 /* ssl mismatches, not ssl by itself*/
-#define CL_SCAN_PHISHING_BLOCKCLOAK 0x1000
+#define CL_SCAN_PHISHING_BLOCKCLOAK	0x1000
-#define CL_SCAN_ELF		    0x2000
+#define CL_SCAN_ELF			0x2000
-#define CL_SCAN_PDF		    0x4000
+#define CL_SCAN_PDF			0x4000
-#define CL_SCAN_STRUCTURED	    0x8000
+#define CL_SCAN_STRUCTURED		0x8000
-
+#define CL_SCAN_STRUCTURED_SSN_NORMAL	0x10000
+#define CL_SCAN_STRUCTURED_SSN_STRIPPED	0x20000
 
 /* recommended scan settings */
 #define CL_SCAN_STDOPT		(CL_SCAN_ARCHIVE | CL_SCAN_MAIL | CL_SCAN_OLE2 | CL_SCAN_HTML | CL_SCAN_PE | CL_SCAN_ALGORITHMIC | CL_SCAN_ELF)
@@ -145,11 +146,6 @@ struct cl_engine {
     void *ignored;
 };
 
-/* Structured data flags */
-#define CL_STRUCTURED_CONF_SSN_BOTH        0x00
-#define CL_STRUCTURED_CONF_SSN_NORMAL      0x01
-#define CL_STRUCTURED_CONF_SSN_STRIPPED    0x02
-
 struct cl_limits {
     unsigned long int maxscansize;  /* during the scanning of archives this size
 				     * will never be exceeded
@@ -167,9 +163,8 @@ struct cl_limits {
      * number of occurences of an CC# or SSN before the system will
      * generate a notification.
      */
-    unsigned long min_cc_count;
+    unsigned int min_cc_count;
-    unsigned long min_ssn_count;
+    unsigned int min_ssn_count;
-    unsigned long structured_flags;
 };
 
 struct cl_stat {

libclamav/dconf.c

@@ -99,6 +99,7 @@ static struct dconf_module modules[] = {
     { "OTHER",	    "RIFF",	    OTHER_CONF_RIFF,	    1 },
     { "OTHER",	    "JPEG",	    OTHER_CONF_JPEG,	    1 },
     { "OTHER",	    "CRYPTFF",	    OTHER_CONF_CRYPTFF,	    1 },
+    { "OTHER",	    "DLP",	    OTHER_CONF_DLP,	    1 },
 
     { "PHISHING",   "ENGINE",       PHISHING_CONF_ENGINE,   1 },
     { "PHISHING",   "ENTCONV",      PHISHING_CONF_ENTCONV,  1 },

libclamav/dconf.h

@@ -89,6 +89,7 @@ struct cli_dconf {
 #define OTHER_CONF_RIFF	    0x4
 #define OTHER_CONF_JPEG	    0x8
 #define OTHER_CONF_CRYPTFF  0x10
+#define OTHER_CONF_DLP	    0x20
 
 /* Phishing flags */
 #define PHISHING_CONF_ENGINE   0x1

libclamav/scanners.c

@@ -1514,36 +1514,38 @@ static int cli_scan_structured(int desc, cli_ctx *ctx)
     else
 	ccfunc = dlp_get_cc_count;
 
-    ssnfunc = dlp_get_ssn_count;;
+    switch((ctx->options & CL_SCAN_STRUCTURED_SSN_NORMAL) | (ctx->options & CL_SCAN_STRUCTURED_SSN_STRIPPED)) {
 
-    switch(lim->structured_flags) {
+	case (CL_SCAN_STRUCTURED_SSN_NORMAL | CL_SCAN_STRUCTURED_SSN_STRIPPED):
-
-	case CL_STRUCTURED_CONF_SSN_BOTH:
 	    if(lim->min_ssn_count == 1)
 		ssnfunc = dlp_has_ssn;
 	    else
 		ssnfunc = dlp_get_ssn_count;
 	    break;
 
-	case CL_STRUCTURED_CONF_SSN_NORMAL:
+	case CL_SCAN_STRUCTURED_SSN_NORMAL:
 	    if(lim->min_ssn_count == 1)
 		ssnfunc = dlp_has_normal_ssn;
 	    else
 		ssnfunc = dlp_get_normal_ssn_count;
 	    break;
 
-	case CL_STRUCTURED_CONF_SSN_STRIPPED:
+	case CL_SCAN_STRUCTURED_SSN_STRIPPED:
 	    if(lim->min_ssn_count == 1)
 		ssnfunc = dlp_has_stripped_ssn;
 	    else
 		ssnfunc = dlp_get_stripped_ssn_count;
 	    break;
+
+	default:
+	    ssnfunc = NULL;
     }
 
-    while(((result = cli_readn(desc, buf, 8191)) > 0) && !done) {
+    while(!done && ((result = cli_readn(desc, buf, 8191)) > 0)) {
 	if((cc_count += ccfunc((const unsigned char *)buf, result)) >= lim->min_cc_count)
 	    done = 1;
-	if((ssn_count += ssnfunc((const unsigned char *)buf, result)) >= lim->min_ssn_count)
+
+	if(ssnfunc && ((ssn_count += ssnfunc((const unsigned char *)buf, result)) >= lim->min_ssn_count))
 	    done = 1;
     }
 
@@ -1990,7 +1992,7 @@ int cli_magic_scandesc(int desc, cli_ctx *ctx)
 	    break;
 
 	case CL_TYPE_TEXT_ASCII:
-	    if(SCAN_STRUCTURED)
+	    if(SCAN_STRUCTURED && (DCONF_OTHER & OTHER_CONF_DLP))
 		/* TODO: consider calling this from cli_scanscript() for
 		 * a normalised text
 		 */

shared/cfgparser.c

@@ -55,6 +55,11 @@ struct cfgoption cfg_options[] = {
     {"PhishingRestrictedScan", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},
     /* end of FP prone options */
     {"DetectPUA", OPT_BOOL, 0, NULL, 0, OPT_CLAMD},
+    {"StructuredDataDetection", OPT_BOOL, 0, NULL, 0, OPT_CLAMD},
+    {"StructuredMinCreditCardCount", OPT_NUM, 1, NULL, 0, OPT_CLAMD},
+    {"StructuredMinSSNCount", OPT_NUM, 1, NULL, 0, OPT_CLAMD},
+    {"StructuredSSNFormatNormal", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},
+    {"StructuredSSNFormatStripped", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},
     {"AlgorithmicDetection", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},
     {"ScanHTML", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},
     {"ScanOLE2", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},