> # test.big: test earth with a biggish model
> 
> source("test.prolog.R")
> library(earth)
Loading required package: Formula
Loading required package: plotmo
Loading required package: plotrix
Loading required package: TeachingDemos
> options(digits=3)
> 
> p <- 100
> n <- 20000 # big enough to cross ten-thousand-cases barrier in plotres and plotmo
> 
> # p <- 100; n <- 10e6 # earth 4.4.0: Error in forward.pass: Out of memory (could not allocate 15 GB)
>                       # ok with nk=21, 42 minutes to build model
>                       # earth 4.6.3: Stopped after two hours, much memory paging
> 
> # p <- 100; n <- 9e6  # windows 64 bit system, 2.9 GHz i7, 32 gig ram, SSD drive:
>                       # earth 4.4.0: Out of memory (could not allocate 15 GB)
>                       # earth 4.6.3: ok (earth now uses .Call instead of .C to invoke ForwardPassR)
>                       #              55 mins to build model
> 
> # p <- 100; n <- 8e6  # 51 minutes to build model, additional 1.5 minutes for plotmo and plotres
> 
> # p <- 2; n <- 60e6   # ok
> 
> # p <- 2; n <- 80e6   # ok (but not enough memory to get leverages)
> #                     # 18 minutes to build model, additional 8 minutes for plotmo and plotres
> #
> # p <- 2; n <- 100e6  # earth 4.6.3: thrashes, interupted after a few hours
> #                     # earth 4.4.0 Error in leaps.setup: Reached total allocation of 32673Mb
> #                     # ok with nk=11, not so much thrashing, 10 minutes
> 
> cat("creating x\n")
creating x
> ran <- function() runif(n, min=-1, max=1)
> x <- matrix(ran(), ncol=1)
> if(p >= 2)
+     x <- cbind(x, ran())
> if(p >= 3)
+     x <- cbind(x, ran())
> if(p >= 4) {
+     # xran saves time generating x, ok because func uses only columns x1, x2, and x3
+     xran <- ran()
+     x <- cbind(x, matrix(xran, nrow=n, ncol=p-3))
+ }
> colnames(x) <- paste("x", 1:ncol(x), sep="")
> func <- function(x) # additive, no interactions
+ {
+     y <- sin(4 * x[,1])
+     if(p > 1)
+         y <- y + x[,2]
+     if(p > 2)
+         y <- y + 2 * x[,3]^2 - 1
+     y
+ }
> cat("creating y\n")
creating y
> y <- func(x)
> cat("testing memory handling when an error (Adjust.endspan = -999)\n")
testing memory handling when an error (Adjust.endspan = -999)
> dummy.allowed <- function(degree, pred, parents) TRUE
> expect.err(try(earth(x, y, trace=1.5, allowed=dummy.allowed, Adjust.endspan = -999)), "Adjust.endspan is -999 but should be between 0 and 10")
x[20000,100] with colnames x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x1...
y[20000,1] with colname y, and values 0.7958, 0.4044, -2.497, -1.05...
maxmem 0.1 GB
malloc    400  B: nUses			nPreds 100 sizeof(int) 4
malloc    804  B: nDegree		nMaxTerms 201 sizeof(int) 4
calloc      4  B: iDirs			nMaxTerms 201 nPreds 100 sizeof(int) 4
malloc    804  B: BoolFullSet		nMaxTerms 201 sizeof(bool) 4
malloc    800  B: sPredNames		LENGTH(SEXP_sPredNames) 100 sizeof(char*) 8
FreeEarth
Error in forward.pass(x, y, yw, weights, trace, degree, penalty, nk, thresh,  : 
  Adjust.endspan is -999 but should be between 0 and 10
Got error as expected from try(earth(x, y, trace = 1.5, allowed = dummy.allowed, Adjust.endspan = -999))
> cat("calling earth\n")
calling earth
> start.time <- proc.time()
> a <- earth(x, y, degree=1, trace=1.5)
x[20000,100] with colnames x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x1...
y[20000,1] with colname y, and values 0.7958, 0.4044, -2.497, -1.05...
maxmem 0.1 GB
malloc    400  B: nUses			nPreds 100 sizeof(int) 4
malloc    804  B: nDegree		nMaxTerms 201 sizeof(int) 4
calloc      4  B: iDirs			nMaxTerms 201 nPreds 100 sizeof(int) 4
malloc    804  B: BoolFullSet		nMaxTerms 201 sizeof(bool) 4
malloc    800  B: sPredNames		LENGTH(SEXP_sPredNames) 100 sizeof(char*) 8
malloc      8 MB: xOrder		nRows 20000 nCols 100 sizeof(int) 4
malloc     31 MB: BetaCacheGlobal	nMaxTerms 201 nMaxTerms 201 nPreds 100 sizeof(double) 8
malloc     31 MB: bxOrth		nCases 20000 nMaxTerms 201 sizeof(double) 8
malloc     31 MB: bxOrthCenteredT	nMaxTerms 201 nCases 20000 sizeof(double) 8
malloc      2 kB: bxOrthMean		nMaxTerms 201 nResp 1 sizeof(double) 8
malloc      8  B: yMean			nResp 1 sizeof(double) 8
Forward pass term 1
malloc      6 kB: Q			nMaxTerms 201 sizeof(tQueue) 32
malloc      6 kB: SortedQ		nMaxTerms 201 sizeof(tQueue) 32
malloc    156 kB: xbx			nCases 20000 sizeof(double) 8
malloc      2 kB: CovSx			nMaxTerms 201 sizeof(double) 8
calloc      8  B: CovCol		nMaxTerms 201 sizeof(double) 8
calloc      8  B: ycboSum		nMaxTerms 201 nResp 1 sizeof(double) 8
malloc      8  B: ybxSum		nResp 1 sizeof(double) 8
Forward pass term 2
malloc    156 kB: xbx			nCases 20000 sizeof(double) 8
malloc      2 kB: CovSx			nMaxTerms 201 sizeof(double) 8
calloc      8  B: CovCol		nMaxTerms 201 sizeof(double) 8
calloc      8  B: ycboSum		nMaxTerms 201 nResp 1 sizeof(double) 8
malloc      8  B: ybxSum		nResp 1 sizeof(double) 8
Forward pass term 4
malloc    156 kB: xbx			nCases 20000 sizeof(double) 8
malloc      2 kB: CovSx			nMaxTerms 201 sizeof(double) 8
calloc      8  B: CovCol		nMaxTerms 201 sizeof(double) 8
calloc      8  B: ycboSum		nMaxTerms 201 nResp 1 sizeof(double) 8
malloc      8  B: ybxSum		nResp 1 sizeof(double) 8
Forward pass term 6
malloc    156 kB: xbx			nCases 20000 sizeof(double) 8
malloc      2 kB: CovSx			nMaxTerms 201 sizeof(double) 8
calloc      8  B: CovCol		nMaxTerms 201 sizeof(double) 8
calloc      8  B: ycboSum		nMaxTerms 201 nResp 1 sizeof(double) 8
malloc      8  B: ybxSum		nResp 1 sizeof(double) 8
Forward pass term 8
malloc    156 kB: xbx			nCases 20000 sizeof(double) 8
malloc      2 kB: CovSx			nMaxTerms 201 sizeof(double) 8
calloc      8  B: CovCol		nMaxTerms 201 sizeof(double) 8
calloc      8  B: ycboSum		nMaxTerms 201 nResp 1 sizeof(double) 8
malloc      8  B: ybxSum		nResp 1 sizeof(double) 8
Forward pass term 10
malloc    156 kB: xbx			nCases 20000 sizeof(double) 8
malloc      2 kB: CovSx			nMaxTerms 201 sizeof(double) 8
calloc      8  B: CovCol		nMaxTerms 201 sizeof(double) 8
calloc      8  B: ycboSum		nMaxTerms 201 nResp 1 sizeof(double) 8
malloc      8  B: ybxSum		nResp 1 sizeof(double) 8
Forward pass term 12
malloc    156 kB: xbx			nCases 20000 sizeof(double) 8
malloc      2 kB: CovSx			nMaxTerms 201 sizeof(double) 8
calloc      8  B: CovCol		nMaxTerms 201 sizeof(double) 8
calloc      8  B: ycboSum		nMaxTerms 201 nResp 1 sizeof(double) 8
malloc      8  B: ybxSum		nResp 1 sizeof(double) 8
Forward pass term 14
malloc    156 kB: xbx			nCases 20000 sizeof(double) 8
malloc      2 kB: CovSx			nMaxTerms 201 sizeof(double) 8
calloc      8  B: CovCol		nMaxTerms 201 sizeof(double) 8
calloc      8  B: ycboSum		nMaxTerms 201 nResp 1 sizeof(double) 8
malloc      8  B: ybxSum		nResp 1 sizeof(double) 8
Forward pass term 16
malloc    156 kB: xbx			nCases 20000 sizeof(double) 8
malloc      2 kB: CovSx			nMaxTerms 201 sizeof(double) 8
calloc      8  B: CovCol		nMaxTerms 201 sizeof(double) 8
calloc      8  B: ycboSum		nMaxTerms 201 nResp 1 sizeof(double) 8
malloc      8  B: ybxSum		nResp 1 sizeof(double) 8
Forward pass term 18
malloc    156 kB: xbx			nCases 20000 sizeof(double) 8
malloc      2 kB: CovSx			nMaxTerms 201 sizeof(double) 8
calloc      8  B: CovCol		nMaxTerms 201 sizeof(double) 8
calloc      8  B: ycboSum		nMaxTerms 201 nResp 1 sizeof(double) 8
malloc      8  B: ybxSum		nResp 1 sizeof(double) 8
Forward pass term 20

RSq changed by less than 0.001 at 19 terms, 13 terms used (DeltaRSq 0.00078)
After forward pass GRSq 0.998 RSq 0.998
malloc    804  B: iPivots		nTerms 201 sizeof(int) 4
malloc      2 MB: xUsed			nCases 20000 nUsedCols 13 sizeof(double) 8
malloc    156 kB: Residuals		nCases 20000 nResp 1 sizeof(double) 8
malloc    104  B: qraux			nUsedCols 13 sizeof(double) 8
malloc      2 MB: work			nCases 20000 nUsedCols 13 sizeof(double) 8
FreeEarth (already free)
Prune backward penalty 2 nprune null: selected 12 of 13 terms, and 3 of 100 preds
After pruning pass GRSq 0.997 RSq 0.997
Getting leverages
> if(interactive())
+     printf("n %g p %g: earth time %.3f seconds (%.3f minutes)\n",
+         n, p,
+         (proc.time() - start.time)[3],
+         (proc.time() - start.time)[3] / 60)
> cat("print(summary(a1)):\n")
print(summary(a1)):
> print(summary(a))
Call: earth(x=x, y=y, trace=1.5, degree=1)

                 coefficients
(Intercept)            -2.096
h(-0.609965-x1)         3.841
h(x1- -0.609965)       -1.557
h(x1- -0.366104)        3.545
h(x1- -0.21304)         1.537
h(x1-0.276572)         -3.138
h(x1-0.477924)         -3.827
h(0.998539-x2)         -1.000
h(x3- -0.57676)         2.162
h(0.0224676-x3)         3.296
h(x3-0.0224676)        -0.974
h(x3-0.530607)          1.866

Selected 12 of 13 terms, and 3 of 100 predictors
Termination condition: RSq changed by less than 0.001 at 13 terms
Importance: x2, x3, x1, x4-unused, x5-unused, x6-unused, x7-unused, ...
Number of terms at each degree of interaction: 1 11 (additive model)
GCV 0.00294    RSS 58.6    GRSq 0.997    RSq 0.997
> invisible(gc())
> cat("calling plotmo\n")
calling plotmo
> plotmo(a, trace=-1)
> invisible(gc())
> cat("calling plotres\n")
calling plotres
> set.seed(2015) # TODO this is necessary, why?
> plot(a, trace=1)
stats::residuals(object=earth.object, type="response")
stats::fitted(object=earth.object)
got model response from getCall(object)$y
using 10000 of 20000 residuals, forcing id.n=0 because of that (implementation restriction)

training rsq 1.00
> if(interactive()) {
+     printf("n %g p %g: total time %.3f seconds (%.3f minutes)\n",
+          n, p,
+         (proc.time() - start.time)[3],
+         (proc.time() - start.time)[3] / 60)
+     x <- y <- 0 # free memory by reducing size of large matrices
+     gc()        # release memory back to operating system
+ }
> source("test.epilog.R")
