v2: Allow partial processing of XML, JSON, and multipart request body

apache2/apache2_io.c

@@ -178,7 +178,7 @@
 /**
  * Reads request body from a client.
  */
 apr_status_t read_request_body(modsec_rec *msr, char **error_msg) {
     assert(msr != NULL);
     assert(error_msg!= NULL);
     request_rec *r = msr->r;
@@ -299,38 +299,39 @@
 #endif
             }
 
+            if (msr->reqbody_length + buflen > (apr_size_t)msr->txcfg->reqbody_limit && msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL) {
+                buflen = (apr_size_t)msr->txcfg->reqbody_limit - msr->reqbody_length;
+                if (msr->txcfg->debuglog_level >= 9) {
+                    msr_log(msr, 9, "Input filter: Bucket type %s shortened by %" APR_SIZE_T_FMT " bytes because of reqbody_limit and ProcessPartial.",
+                            bucket->type->name, (apr_size_t)msr->txcfg->reqbody_limit - msr->reqbody_length);
+                }
+
+                finished_reading = 1;
+                modsecurity_request_body_enable_partial_processing(msr);
+            } else if (   (msr->msc_reqbody_no_files_length > (unsigned long)msr->txcfg->reqbody_no_files_limit)
+                    && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL))
+            {
+                if (msr->txcfg->debuglog_level >= 9) {
+                    msr_log(msr, 9, "Input filter: Bucket type %s skip storing because of no_files_limit and ProcessPartial.",
+                            bucket->type->name);
+                }
+                buflen = 0;
+                finished_reading = 1;
+            }
+
             msr->reqbody_length += buflen;
 
             if (buflen != 0) {
                 int rcbs = modsecurity_request_body_store(msr, buf, buflen, error_msg);
-
-                if (msr->reqbody_length > (apr_size_t)msr->txcfg->reqbody_limit && msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL) {
-                    finished_reading = 1;
-                }
-
                 if (rcbs < 0) {
                     if (rcbs == -5) {
-                        if((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_REJECT)) {
-                            *error_msg = apr_psprintf(msr->mp, "Request body no files data length is larger than the "
-                                    "configured limit (%ld).", msr->txcfg->reqbody_no_files_limit);
-                            return HTTP_REQUEST_ENTITY_TOO_LARGE;
-                        } else if ((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL)) {
-                            *error_msg = apr_psprintf(msr->mp, "Request body no files data length is larger than the "
-                                    "configured limit (%ld).", msr->txcfg->reqbody_no_files_limit);
-                        } else if ((msr->txcfg->is_enabled == MODSEC_DETECTION_ONLY) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL)) {
-                            *error_msg = apr_psprintf(msr->mp, "Request body no files data length is larger than the "
-                                    "configured limit (%ld).", msr->txcfg->reqbody_no_files_limit);
-                        } else {
-                            *error_msg = apr_psprintf(msr->mp, "Request body no files data length is larger than the "
-                                    "configured limit (%ld).", msr->txcfg->reqbody_no_files_limit);
-                            return HTTP_REQUEST_ENTITY_TOO_LARGE;
-                        }
+                        return HTTP_REQUEST_ENTITY_TOO_LARGE;
                     }
 
-                    if((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_REJECT))
+                    if ((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_REJECT)) {
                         return HTTP_INTERNAL_SERVER_ERROR;
+                    }
                 }
-
             }
 
             if (APR_BUCKET_IS_EOS(bucket)) {
@@ -351,11 +352,14 @@
 
     msr->if_status = IF_STATUS_WANTS_TO_RUN;
 
-    if (rcbe == -5) {
-        return HTTP_REQUEST_ENTITY_TOO_LARGE;
-    }
     if (rcbe < 0) {
-        return HTTP_INTERNAL_SERVER_ERROR;
+        if (rcbe == -5) {
+            return HTTP_REQUEST_ENTITY_TOO_LARGE;
+        }
+
+        if ((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_REJECT)) {
+            return HTTP_INTERNAL_SERVER_ERROR;
+        }
     }
     return APR_SUCCESS;
 }

apache2/modsecurity.h

@@ -279,6 +279,7 @@ struct modsec_rec {
     unsigned int         if_started_forwarding;
 
     apr_size_t           reqbody_length;
+    unsigned int         reqbody_partial_processing_enabled;
 
     apr_bucket_brigade  *of_brigade;
     unsigned int         of_status;
@@ -736,6 +737,8 @@ apr_status_t DSOLOCAL modsecurity_request_body_start(modsec_rec *msr, char **err
 apr_status_t DSOLOCAL modsecurity_request_body_store(modsec_rec *msr,
     const char *data, apr_size_t length, char **error_msg);
 
+void DSOLOCAL modsecurity_request_body_enable_partial_processing(modsec_rec *msr);
+
 apr_status_t DSOLOCAL modsecurity_request_body_end(modsec_rec *msr, char **error_msg);
 
 apr_status_t DSOLOCAL modsecurity_request_body_to_stream(modsec_rec *msr, const char *buffer, int buflen, char **error_msg);

apache2/msc_json.c

@@ -187,7 +187,7 @@ static int yajl_start_array(void *ctx) {
     msr->json->current_depth++;
     if (msr->json->current_depth > msr->txcfg->reqbody_json_depth_limit) {
         msr->json->depth_limit_exceeded = 1;
-	return 0;
+        return 0;
     }
 
     if (msr->txcfg->debuglog_level >= 9) {
@@ -262,7 +262,7 @@ static int yajl_start_map(void *ctx)
     msr->json->current_depth++;
     if (msr->json->current_depth > msr->txcfg->reqbody_json_depth_limit) {
         msr->json->depth_limit_exceeded = 1;
-	return 0;
+        return 0;
     }
 
     if (msr->txcfg->debuglog_level >= 9) {
@@ -367,6 +367,10 @@ int json_init(modsec_rec *msr, char **error_msg) {
     return 1;
 }
 
+void json_allow_partial_values(modsec_rec *msr) {
+    (void)yajl_config(msr->json->handle, yajl_allow_partial_values, 1);
+}
+
 /**
  * Feed one chunk of data to the JSON parser.
  */
@@ -380,16 +384,16 @@ int json_process_chunk(modsec_rec *msr, const char *buf, unsigned int size, char
     /* Feed our parser and catch any errors */
     msr->json->status = yajl_parse(msr->json->handle, buf, size);
     if (msr->json->status != yajl_status_ok) {
-	if (msr->json->depth_limit_exceeded) {
-           *error_msg = "JSON depth limit exceeded";
-	} else {
-        if (msr->json->yajl_error) *error_msg = msr->json->yajl_error;
-        else {
-             char* yajl_err = yajl_get_error(msr->json->handle, 0, buf, size);
-             *error_msg = apr_pstrdup(msr->mp, yajl_err);
-             yajl_free_error(msr->json->handle, yajl_err);
+        if (msr->json->depth_limit_exceeded) {
+             *error_msg = "JSON depth limit exceeded";
+        } else {
+            if (msr->json->yajl_error) *error_msg = msr->json->yajl_error;
+            else {
+                 char* yajl_err = yajl_get_error(msr->json->handle, 0, buf, size);
+                 *error_msg = apr_pstrdup(msr->mp, yajl_err);
+                 yajl_free_error(msr->json->handle, yajl_err);
+            }
         }
-	}
         return -1;
     }
 
@@ -409,13 +413,13 @@ int json_complete(modsec_rec *msr, char **error_msg) {
     /* Wrap up the parsing process */
     msr->json->status = yajl_complete_parse(msr->json->handle);
     if (msr->json->status != yajl_status_ok) {
-	if (msr->json->depth_limit_exceeded) {
-           *error_msg = "JSON depth limit exceeded";
-	} else {
-           char *yajl_err = yajl_get_error(msr->json->handle, 0, NULL, 0);
-           *error_msg = apr_pstrdup(msr->mp, yajl_err);
-           yajl_free_error(msr->json->handle, yajl_err);
-	}
+        if (msr->json->depth_limit_exceeded) {
+             *error_msg = "JSON depth limit exceeded";
+        } else {
+             char *yajl_err = yajl_get_error(msr->json->handle, 0, NULL, 0);
+             *error_msg = apr_pstrdup(msr->mp, yajl_err);
+             yajl_free_error(msr->json->handle, yajl_err);
+        }
 
         return -1;
     }

apache2/msc_json.h

@@ -48,6 +48,8 @@ struct json_data {
 
 int DSOLOCAL json_init(modsec_rec *msr, char **error_msg);
 
+void DSOLOCAL json_allow_partial_values(modsec_rec *msr);
+
 int DSOLOCAL json_process(modsec_rec *msr, const char *buf,
     unsigned int size, char **error_msg);
 

apache2/msc_multipart.c

@@ -251,6 +251,23 @@
     return 1;
 }
 
+/* Enable partial processing if no_files_len exceeds limit and action is ProcessPartial */
+static void modsecurity_request_body_enable_partial_processing_for_no_files_length(modsec_rec *msr,
+    int *length)
+{
+    /* Enable partial processing if no_files_len exceeds limit and action is ProcessPartial */
+    if (   (msr->msc_reqbody_no_files_length > (unsigned long)msr->txcfg->reqbody_no_files_limit)
+        && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL))
+    {
+        *length -= msr->msc_reqbody_no_files_length - (unsigned long)msr->txcfg->reqbody_no_files_limit;
+        if (msr->txcfg->debuglog_level >= 9) {
+            msr_log(msr, 9, "MULTIPART: length shortened by %" APR_SIZE_T_FMT " bytes because of no_files_len limit.",
+                    msr->msc_reqbody_no_files_length - (unsigned long)msr->txcfg->reqbody_no_files_limit);
+        }
+        modsecurity_request_body_enable_partial_processing(msr);
+    }
+}
+
 /**
  *
  */
@@ -272,6 +289,7 @@
 
     /* The buffer is data so increase the data length counter. */
     msr->msc_reqbody_no_files_length += (MULTIPART_BUF_SIZE - msr->mpd->bufleft);
+    modsecurity_request_body_enable_partial_processing_for_no_files_length(msr, &msr->mpd->bufleft);
 
     if (len > 1) {
         if (msr->mpd->buf[len - 2] == '\r') {
@@ -421,7 +439,7 @@
             if (data == msr->mpd->buf) {
                 *error_msg = apr_psprintf(msr->mp, "Multipart: Invalid part header (header name missing).");
 
-                 return -1;
+                return -1;
             }
 
             /* check if multipart header contains any invalid characters */
@@ -586,6 +604,7 @@
 
         /* The buffer contains data so increase the data length counter. */
         msr->msc_reqbody_no_files_length += (MULTIPART_BUF_SIZE - msr->mpd->bufleft) + msr->mpd->reserve[0];
+        modsecurity_request_body_enable_partial_processing_for_no_files_length(msr, &msr->mpd->bufleft);
 
         /* add this part to the list of parts */
 
@@ -967,7 +986,7 @@
  * Finalise multipart processing. This method is invoked at the end, when it
  * is clear that there is no more data to be processed.
  */
 int multipart_complete(modsec_rec *msr, char **error_msg) {
     assert(msr != NULL);
     assert(error_msg != NULL);
     if (msr->mpd == NULL) return 1;
@@ -1023,38 +1042,94 @@
              * processed yet) in the buffer.
              */
             if (msr->mpd->buf_contains_line) {
-                if ( ((unsigned int)(MULTIPART_BUF_SIZE - msr->mpd->bufleft) == (4 + strlen(msr->mpd->boundary)))
+                /*
+                 * Note that the buffer may end with the final boundary followed by only CR,
+                 * coming from the [CRLF epilogue], when allow_process_partial == 1 (which is
+                 * set when SecRequestBodyLimitAction is ProcessPartial and the request body
+                 * length exceeds SecRequestBodyLimit).
+                 *
+                 * The following definitions are copied from RFC 2046:
+                 *
+                 * dash-boundary := "--" boundary
+                 *
+                 * delimiter := CRLF dash-boundary
+                 *
+                 * close-delimiter := delimiter "--"
+                 *
+                 * multipart-body := [preamble CRLF]
+                 *                   dash-boundary transport-padding CRLF
+                 *                   body-part *encapsulation
+                 *                   close-delimiter transport-padding
+                 *                   [CRLF epilogue]
+                 */
+                unsigned int buf_data_len = (unsigned int)(MULTIPART_BUF_SIZE - msr->mpd->bufleft);
+                size_t boundary_len = strlen(msr->mpd->boundary);
+                if ( (buf_data_len >= 2 + boundary_len)
                     && (*(msr->mpd->buf) == '-')
                     && (*(msr->mpd->buf + 1) == '-')
-                    && (strncmp(msr->mpd->buf + 2, msr->mpd->boundary, strlen(msr->mpd->boundary)) == 0)
-                    && (*(msr->mpd->buf + 2 + strlen(msr->mpd->boundary)) == '-')
-                    && (*(msr->mpd->buf + 2 + strlen(msr->mpd->boundary) + 1) == '-') )
+                    && (strncmp(msr->mpd->buf + 2, msr->mpd->boundary, boundary_len) == 0) )
                 {
-                    if ((msr->mpd->crlf_state_buf_end == 2) && (msr->mpd->flag_lf_line != 1)) {
-                        msr->mpd->flag_lf_line = 1;
-                        if (msr->mpd->flag_crlf_line) {
-                            msr_log(msr, 4, "Multipart: Warning: mixed line endings used (CRLF/LF).");
-                        } else {
-                            msr_log(msr, 4, "Multipart: Warning: incorrect line endings used (LF).");
+                    if ( (buf_data_len >= 2 + boundary_len + 2)
+                        && (*(msr->mpd->buf + 2 + boundary_len) == '-')
+                        && (*(msr->mpd->buf + 2 + boundary_len + 1) == '-') )
+                    {
+                        /* If body fits in limit and ends with final boundary plus just CR, reject it. */
+                        if ( (msr->mpd->allow_process_partial == 0)
+                            && (buf_data_len == 2 + boundary_len + 2 + 1)
+                            && (*(msr->mpd->buf + 2 + boundary_len + 2) == '\r') )
+                        {
+                            *error_msg = apr_psprintf(msr->mp, "Multipart: Invalid epilogue after final boundary.");
+                            return -1;
                         }
+
+                        if ((msr->mpd->crlf_state_buf_end == 2) && (msr->mpd->flag_lf_line != 1)) {
+                            msr->mpd->flag_lf_line = 1;
+                            if (msr->mpd->flag_crlf_line) {
+                                msr_log(msr, 4, "Multipart: Warning: mixed line endings used (CRLF/LF).");
+                            } else {
+                                msr_log(msr, 4, "Multipart: Warning: incorrect line endings used (LF).");
+                            }
+                        }
+                        if (msr->mpd->mpp_substate_part_data_read == 0) {
+                            /* it looks like the final boundary, but it's where part data should begin */
+                            msr->mpd->flag_invalid_part = 1;
+                            msr_log(msr, 4, "Multipart: Warning: Invalid part (data contains final boundary)");
+                        }
+                        /* Looks like the final boundary - process it. */
+                        if (multipart_process_boundary(msr, 1 /* final */, error_msg) < 0) {
+                            msr->mpd->flag_error = 1;
+                            return -1;
+                        }
+
+                        /* The payload is complete after all. */
+                        msr->mpd->is_complete = 1;
                     }
-                    if (msr->mpd->mpp_substate_part_data_read == 0) {
-                        /* it looks like the final boundary, but it's where part data should begin */
-                        msr->mpd->flag_invalid_part = 1;
-                        msr_log(msr, 4, "Multipart: Warning: Invalid part (data contains final boundary)");
-                    }
-                    /* Looks like the final boundary - process it. */
-                    if (multipart_process_boundary(msr, 1 /* final */, error_msg) < 0) {
-                        msr->mpd->flag_error = 1;
-                        return -1;
+                    else if (msr->mpd->allow_process_partial == 1) {
+                        if (buf_data_len >= 2 + boundary_len + 1) {
+                            if (*(msr->mpd->buf + 2 + boundary_len) == '-') {
+                                if ( (buf_data_len >= 2 + boundary_len + 2)
+                                    && (*(msr->mpd->buf + 2 + boundary_len + 1) != '-') ) {
+                                    *error_msg = apr_psprintf(msr->mp, "Multipart: Invalid final boundary.");
+                                    return -1;
+                                }
+                            }
+                            else if ( (*(msr->mpd->buf + 2 + boundary_len) != '\r')
+                                || ((buf_data_len >= 2 + boundary_len + 2)
+                                    && (*(msr->mpd->buf + 2 + boundary_len + 1) != '\n')) ) {
+                                *error_msg = apr_psprintf(msr->mp, "Multipart: Invalid boundary.");
+                                return -1;
+                            }
+                        }
+                        /* process it as a non-final boundary to avoid building a new part. */
+                        if (multipart_process_boundary(msr, 0, error_msg) < 0) {
+                            msr->mpd->flag_error = 1;
+                            return -1;
+                        }
                     }
-
-                    /* The payload is complete after all. */
-                    msr->mpd->is_complete = 1;
                 }
             }
 
-            if (msr->mpd->is_complete == 0) {
+            if (msr->mpd->is_complete == 0 && msr->mpd->allow_process_partial == 0) {
                 *error_msg = apr_psprintf(msr->mp, "Multipart: Final boundary missing.");
                 return -1;
             }
@@ -1296,10 +1371,10 @@
             if (c == 0x0a) {
                 if (msr->mpd->crlf_state == 1) {
                     msr->mpd->crlf_state = 3;
-	        } else {
+                } else {
                     msr->mpd->crlf_state = 2;
-	        }
-	    }
+                }
+            }
             msr->mpd->crlf_state_buf_end = msr->mpd->crlf_state;
         }
 

apache2/msc_multipart.h

@@ -124,6 +124,7 @@ struct multipart_data {
 
     int                      seen_data;
     int                      is_complete;
+    int                      allow_process_partial;
 
     int                      flag_error;
     int                      flag_data_before;

apache2/msc_parsers.c

@@ -240,7 +240,7 @@
 /**
  *
  */
 int parse_arguments(modsec_rec *msr, const char *s, apr_size_t inputlength,
         int argument_separator, const char *origin,
         apr_table_t *arguments, int *invalid_count)
 {
@@ -313,7 +313,7 @@
                 value = &buf[j];
             }
         }
-        else {
+        else if (i < inputlength || msr->reqbody_partial_processing_enabled == 0) {
             arg->value_len = urldecode_nonstrict_inplace_ex((unsigned char *)value, arg->value_origin_len, invalid_count, &changed);
             arg->value = apr_pstrmemdup(msr->mp, value, arg->value_len);
 
@@ -330,7 +330,7 @@
     }
 
     /* the last parameter was empty */
-    if (status == 1) {
+    if (status == 1 && msr->reqbody_partial_processing_enabled == 0) {
         arg->value_len = 0;
         arg->value = "";