Fix frollapply to handle n > len case

NEWS.md

@@ -46,6 +46,8 @@
 
 7. `fread()` can now read from connections directly by spilling to a temporary file first, [#561](https://github.com/Rdatatable/data.table/issues/561). For the best throughput, point `tmpdir=` (or the global temp directory) to fast storage like an SSD or RAM. Thanks to Chris Neff for the report and @ben-schwen for the implementation.
 
+8. `frollapply()` no longer produces output longer than the input when the window length is also longer than the input [#7646](https://github.com/Rdatatable/data.table/issues/7646). Thanks to @hadley-johnson for reporting and @jangorecki for the fix.
+
 ### Notes
 
 1. {data.table} now depends on R 3.5.0 (2018).

R/frollapply.R

@@ -278,8 +278,10 @@ frollapply = function(X, N, FUN, ..., by.column=TRUE, fill=NA, align=c("right","
     cpy = copy
     ansMask = function(len, n) {
       mask = rep(TRUE, len)
-      if (n) ## handle n==0
+      if (n) { ## handle n==0
+        n = min(n, len + 1L) # cap at len (bugfix #7646)
         mask[seq_len(n-1L)] = FALSE
+      }
       mask
     }
     tight0 = function(i, dest, src, n) FUN(dest, ...) ## skip memcpy when n==0

inst/tests/froll.Rraw

@@ -2317,3 +2317,6 @@ test(6015.907, frolladapt(c(1L,3L,2L), 2L), error="be sorted, have no duplicates
 test(6015.908, frolladapt(c(1L,2L,2L), 2L), error="be sorted, have no duplicates, have no NAs")
 test(6015.909, frolladapt(c(1L,2L,NA_integer_), 2L), error="be sorted, have no duplicates, have no NAs") ## loop that checks for sorted will detect NAs as well, except for first element
 test(6015.910, frolladapt(c(NA_integer_,1L,2L), 2L), error="be sorted, have no duplicates, have no NAs") ## first NA is detected by extra check
+
+# frollapply can produce output longer than the input when the window-length is also longer than the input #7646
+test(6100.1, frollapply(c(1,2,3,4,5),N=7,FUN=sum), rep(NA, 5L))